swagger: '2.0' info: title: estela API v1.0 Documentation description: estela API Swagger Specification version: v1 host: 127.0.0.1:8000 schemes: - http basePath: / consumes: - application/json produces: - application/json securityDefinitions: Basic: type: basic security: - Basic: [] paths: /api/account/changePassword/change: patch: operationId: api_account_changePassword_change description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ChangePassword' responses: '200': description: '' schema: $ref: '#/definitions/Token' tags: - api parameters: [] /api/account/resetPassword/confirm: patch: operationId: api_account_resetPassword_confirm description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ResetPasswordConfirm' - name: token in: query description: Token required: true type: string - name: pair in: query description: Pair required: true type: string responses: '200': description: '' schema: $ref: '#/definitions/Token' '401': description: '' schema: type: object properties: error: type: string tags: - api parameters: [] /api/account/resetPassword/request: post: operationId: api_account_resetPassword_request description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ResetPasswordRequest' responses: '201': description: '' schema: $ref: '#/definitions/ResetPasswordRequest' tags: - api parameters: [] /api/account/resetPassword/validate: get: operationId: api_account_resetPassword_validate description: '' parameters: - name: token in: query description: Token required: true type: string - name: pair in: query description: Pair required: true type: string responses: '200': description: '' schema: type: object properties: message: type: string '400': description: '' schema: type: object properties: message: type: string tags: - api parameters: [] /api/auth/activate: get: operationId: api_auth_activate description: '' parameters: [] responses: '200': description: '' schema: type: array items: $ref: '#/definitions/AuthToken' tags: - api parameters: [] /api/auth/login: post: operationId: api_auth_login description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Login' responses: '200': description: '' schema: $ref: '#/definitions/Token' tags: - api parameters: [] /api/auth/profile: get: operationId: api_auth_profile_list description: '' parameters: [] responses: '200': description: '' schema: type: array items: $ref: '#/definitions/UserProfile' tags: - api post: operationId: api_auth_profile_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/UserProfile' responses: '201': description: '' schema: $ref: '#/definitions/UserProfile' tags: - api parameters: [] /api/auth/profile/{username}: get: operationId: api_auth_profile_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/UserProfile' tags: - api put: operationId: api_auth_profile_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/UserProfile' responses: '200': description: '' schema: $ref: '#/definitions/UserProfile' tags: - api patch: operationId: api_auth_profile_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/UserProfile' responses: '200': description: '' schema: $ref: '#/definitions/UserProfile' tags: - api delete: operationId: api_auth_profile_delete description: '' parameters: [] responses: '204': description: '' tags: - api parameters: - name: username in: path description: Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only. required: true type: string pattern: ^[\w.@+-]+$ /api/auth/register: post: operationId: api_auth_register description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/User' responses: '200': description: '' schema: $ref: '#/definitions/Token' tags: - api parameters: [] /api/notifications: get: operationId: api_notifications_list description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/Notification' tags: - api post: operationId: api_notifications_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Notification' responses: '201': description: '' schema: $ref: '#/definitions/Notification' tags: - api parameters: [] /api/notifications/{nid}: get: operationId: api_notifications_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/Notification' tags: - api put: operationId: api_notifications_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/NotificationUpdate' responses: '200': description: '' schema: $ref: '#/definitions/NotificationUpdate' tags: - api patch: operationId: api_notifications_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Notification' responses: '200': description: '' schema: $ref: '#/definitions/Notification' tags: - api delete: operationId: api_notifications_delete description: '' parameters: [] responses: '204': description: '' tags: - api parameters: - name: nid in: path description: A unique integer value identifying each notification required: true type: integer /api/projects: get: operationId: api_projects_list description: '' parameters: - name: page in: query required: false type: integer - name: page_size in: query required: false type: integer - name: search in: query required: false type: string - name: ordering in: query description: 'Order by: name, category, framework, created, last_modified, role (prefix with - for desc)' required: false type: string responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/Project' tags: - api post: operationId: api_projects_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Project' responses: '201': description: '' schema: $ref: '#/definitions/Project' tags: - api parameters: [] /api/projects/{pid}: get: operationId: api_projects_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/Project' tags: - api put: operationId: api_projects_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ProjectUpdate' responses: '200': description: '' schema: $ref: '#/definitions/ProjectUpdate' tags: - api patch: operationId: api_projects_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Project' responses: '200': description: '' schema: $ref: '#/definitions/Project' tags: - api delete: operationId: api_projects_delete description: '' parameters: [] responses: '204': description: Project deleted tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/activities: get: operationId: api_projects_activities description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: number - name: page_size in: query description: Number of results to return per page. required: false type: number responses: '200': description: '' schema: $ref: '#/definitions/ProjectActivity' tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/cronjobs: get: operationId: api_projects_cronjobs description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: number - name: page_size in: query description: Number of results to return per page. required: false type: number responses: '200': description: '' schema: $ref: '#/definitions/ProjectCronJob' tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/current_usage: get: operationId: api_projects_current_usage description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/ProjectUsage' tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/deploys: get: operationId: api_projects_deploys_list description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/Deploy' tags: - api post: operationId: api_projects_deploys_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/DeployCreate' responses: '201': description: '' schema: $ref: '#/definitions/DeployCreate' tags: - api parameters: - name: pid in: path required: true type: string /api/projects/{pid}/deploys/{did}: get: operationId: api_projects_deploys_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/Deploy' tags: - api put: operationId: api_projects_deploys_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/DeployUpdate' responses: '200': description: '' schema: $ref: '#/definitions/DeployUpdate' tags: - api patch: operationId: api_projects_deploys_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Deploy' responses: '200': description: '' schema: $ref: '#/definitions/Deploy' tags: - api delete: operationId: api_projects_deploys_delete description: '' parameters: [] responses: '204': description: '' tags: - api parameters: - name: did in: path description: A unique integer value identifying this deploy. required: true type: integer - name: pid in: path required: true type: string /api/projects/{pid}/deploys/{did}/logs: get: operationId: api_projects_deploys_logs description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/Deploy' tags: - api parameters: - name: did in: path description: A unique integer value identifying this deploy. required: true type: integer - name: pid in: path required: true type: string /api/projects/{pid}/jobs: get: operationId: api_projects_jobs description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: number - name: page_size in: query description: Number of results to return per page. required: false type: number responses: '200': description: '' schema: $ref: '#/definitions/ProjectJob' tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/resource_status: get: operationId: api_projects_resource_status description: '' parameters: [] responses: '200': description: '' schema: type: object properties: memory_used: description: Memory currently in use (Mi). type: number memory_quota: description: Owner's total memory quota across all their projects (Mi). type: number used_pct: description: Percentage of quota in use. type: number tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/projects/{pid}/spiders: get: operationId: api_projects_spiders_list description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: search in: query description: Filter spiders by name (case-insensitive) required: false type: string responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/Spider' tags: - api parameters: - name: pid in: path required: true type: string /api/projects/{pid}/spiders/{sid}: get: operationId: api_projects_spiders_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/Spider' tags: - api put: operationId: api_projects_spiders_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderUpdate' responses: '200': description: '' schema: $ref: '#/definitions/SpiderUpdate' tags: - api patch: operationId: api_projects_spiders_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/Spider' responses: '200': description: '' schema: $ref: '#/definitions/Spider' tags: - api parameters: - name: pid in: path required: true type: string - name: sid in: path description: A unique integer value identifying this spider. required: true type: integer /api/projects/{pid}/spiders/{sid}/cronjobs: get: operationId: api_projects_spiders_cronjobs_list description: '' parameters: - name: tag in: query description: Cron job tag. required: false type: string - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/SpiderCronJob' tags: - api post: operationId: api_projects_spiders_cronjobs_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderCronJobCreate' responses: '201': description: '' schema: $ref: '#/definitions/SpiderCronJobCreate' tags: - api parameters: - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/cronjobs/{cjid}: get: operationId: api_projects_spiders_cronjobs_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/SpiderCronJob' tags: - api put: operationId: api_projects_spiders_cronjobs_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderCronJobUpdate' responses: '200': description: '' schema: $ref: '#/definitions/SpiderCronJobUpdate' tags: - api patch: operationId: api_projects_spiders_cronjobs_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderCronJob' responses: '200': description: '' schema: $ref: '#/definitions/SpiderCronJob' tags: - api delete: operationId: api_projects_spiders_cronjobs_delete description: '' parameters: [] responses: '204': description: Cronjob deleted tags: - api parameters: - name: cjid in: path description: A unique integer value identifying this cron job. required: true type: integer - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/cronjobs/{cjid}/run_once: get: operationId: api_projects_spiders_cronjobs_run_once description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/SpiderCronJob' tags: - api parameters: - name: cjid in: path description: A unique integer value identifying this cron job. required: true type: integer - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs: get: operationId: api_projects_spiders_jobs_list description: '' parameters: - name: cronjob in: query description: Related cron job. required: false type: number - name: status in: query description: Job status. required: false type: string - name: tag in: query description: Job tag. required: false type: string - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/SpiderJob' tags: - api post: operationId: api_projects_spiders_jobs_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderJobCreate' - name: sync in: query description: True to bypass the dispatch queue and send directly to K8s. type: boolean responses: '201': description: '' schema: $ref: '#/definitions/SpiderJobCreate' tags: - api parameters: - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs/{jid}: get: operationId: api_projects_spiders_jobs_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/SpiderJob' tags: - api put: operationId: api_projects_spiders_jobs_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderJobUpdate' responses: '200': description: '' schema: $ref: '#/definitions/SpiderJobUpdate' tags: - api patch: operationId: api_projects_spiders_jobs_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/SpiderJob' responses: '200': description: '' schema: $ref: '#/definitions/SpiderJob' tags: - api parameters: - name: jid in: path description: A unique integer value identifying this job. required: true type: integer - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs/{jid}/data: get: operationId: api_projects_spiders_jobs_data_list description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: type in: query description: Spider job data type. required: false type: string - name: search in: query description: Search term to filter log entries. required: false type: string - name: level in: query description: Log level filter (DEBUG, INFO, WARNING, ERROR, CRITICAL). required: false type: string responses: '200': description: '' schema: required: - count - result type: object properties: count: description: Data items count. type: number previous: description: URI to the previous data chunk. type: string format: uri x-nullable: true next: description: URI to the next data chunk. type: string format: uri x-nullable: true results: description: Data items. type: array items: type: object tags: - api parameters: - name: jid in: path required: true type: string - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs/{jid}/data/delete: post: operationId: api_projects_spiders_jobs_data_delete description: '' parameters: - name: type in: query description: Spider job data type. required: true type: string responses: '200': description: Deletion successful '404': description: Could not delete data tags: - api parameters: - name: jid in: path required: true type: string - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs/{jid}/data/download: get: operationId: api_projects_spiders_jobs_data_download description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: type in: query description: Spider job data type. required: false type: string responses: '200': description: '' schema: required: - results type: object properties: results: description: Data items. type: array items: type: object tags: - api parameters: - name: jid in: path required: true type: string - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/spiders/{sid}/jobs/{jid}/error_logs: get: operationId: api_projects_spiders_jobs_error_logs description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/SpiderJob' tags: - api parameters: - name: jid in: path description: A unique integer value identifying this job. required: true type: integer - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/projects/{pid}/usage: get: operationId: api_projects_usage description: '' parameters: - name: start_date in: query description: Start of date range. required: false type: string - name: end_date in: query description: End of date range. required: false type: string responses: '200': description: '' schema: type: array items: $ref: '#/definitions/UsageRecord' tags: - api parameters: - name: pid in: path description: A UUID identifying this project. required: true type: string format: uuid /api/proxy_provider: get: operationId: api_proxy_provider_list description: '' parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer responses: '200': description: '' schema: required: - count - results type: object properties: count: type: integer next: type: string format: uri x-nullable: true previous: type: string format: uri x-nullable: true results: type: array items: $ref: '#/definitions/ProxyProvider' tags: - api post: operationId: api_proxy_provider_create description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ProxyProvider' responses: '201': description: '' schema: $ref: '#/definitions/ProxyProvider' tags: - api parameters: [] /api/proxy_provider/{proxyid}: get: operationId: api_proxy_provider_read description: '' parameters: [] responses: '200': description: '' schema: $ref: '#/definitions/ProxyProvider' tags: - api put: operationId: api_proxy_provider_update description: In the request we should specify spider, project or job parameters: - name: data in: body required: true schema: $ref: '#/definitions/ProxyProviderUpdate' responses: '200': description: '' schema: $ref: '#/definitions/ProxyProviderResponse' tags: - api patch: operationId: api_proxy_provider_partial_update description: '' parameters: - name: data in: body required: true schema: $ref: '#/definitions/ProxyProvider' responses: '200': description: '' schema: $ref: '#/definitions/ProxyProvider' tags: - api delete: operationId: api_proxy_provider_delete description: '' parameters: [] responses: '204': description: '' tags: - api parameters: - name: proxyid in: path description: A unique integer value identifying this proxy. required: true type: integer /api/stats/{pid}: get: operationId: api_stats_list description: Retrieve stats of all jobs in a range of time, dates must have the format YYYY-mm-dd. parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: start_date in: query description: Start of date in format [%Y-%m-%d] (e.g. 2023-04-01). required: true type: string - name: end_date in: query description: End of date in format [%Y-%m-%d] (e.g. 2023-06-02). required: true type: string - name: offset in: query description: Offset between your local timezone and UTC in 'minutes'. required: false type: integer responses: '200': description: Global stats array with stats summary for each date schema: type: array items: $ref: '#/definitions/ProjectStats' tags: - api parameters: - name: pid in: path required: true type: string /api/stats/{pid}/spider/{sid}: get: operationId: api_stats_spider_list description: Retrieve stats of all spider jobs in a time range, dates must be in YYYY-mm-dd format. parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: start_date in: query description: Start of date in format [%Y-%m-%d] (e.g. 2023-04-01). required: true type: string - name: end_date in: query description: End of date in format [%Y-%m-%d] (e.g. 2023-06-02). required: true type: string - name: offset in: query description: Offset between your local timezone and UTC in 'minutes'. required: false type: integer responses: '200': description: Spiders/Jobs stats array with stats summary for each date schema: type: array items: $ref: '#/definitions/SpidersStats' tags: - api parameters: - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/stats/{pid}/spider/{sid}/jobs: get: operationId: api_stats_spider_jobs description: Retrieve all the jobs of a spider executed in a range of dates. parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: start_date in: query description: Start of date in format [%Y-%m-%d] (e.g. 2023-04-01). required: true type: string - name: end_date in: query description: End of date in format [%Y-%m-%d] (e.g. 2023-06-02). required: true type: string - name: offset in: query description: Offset between your local timezone and UTC in 'minutes'. required: false type: integer responses: '200': description: Paginated jobs belonging to a spider in a range of time schema: $ref: '#/definitions/JobsPagination' tags: - api parameters: - name: pid in: path required: true type: string - name: sid in: path required: true type: string /api/stats/{pid}/spiders: get: operationId: api_stats_spiders description: Retrieve all the spiders executed in a range of dates. parameters: - name: page in: query description: A page number within the paginated result set. required: false type: integer - name: page_size in: query description: Number of results to return per page. required: false type: integer - name: start_date in: query description: Start of date in format [%Y-%m-%d] (e.g. 2023-04-01). required: true type: string - name: end_date in: query description: End of date in format [%Y-%m-%d] (e.g. 2023-06-02). required: true type: string - name: offset in: query description: Offset between your local timezone and UTC in 'minutes'. required: false type: integer responses: '200': description: Paginated spiders launched in a range of time schema: $ref: '#/definitions/SpidersPagination' tags: - api parameters: - name: pid in: path required: true type: string /api/v1/metering/report: post: operationId: api_v1_metering_report description: Append one metered usage fact. Deduplicated by ``idempotency_key``. parameters: - name: data in: body required: true schema: $ref: '#/definitions/MeteringReport' responses: '201': description: '' schema: $ref: '#/definitions/MeteringReportResponse' '200': description: '' schema: $ref: '#/definitions/MeteringReportResponse' tags: - api parameters: [] definitions: ChangePassword: required: - new_password - confirm_new_password - old_password type: object properties: new_password: title: New password type: string minLength: 1 confirm_new_password: title: Confirm new password type: string minLength: 1 old_password: title: Old password type: string minLength: 1 UserDetail: description: User details. required: - username type: object properties: username: title: Username description: Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only. type: string pattern: ^[\w.@+-]+$ maxLength: 150 minLength: 1 email: title: Email address type: string format: email maxLength: 254 Token: required: - key type: object properties: user: $ref: '#/definitions/UserDetail' key: title: Key description: User's auth token key. type: string maxLength: 40 minLength: 1 ResetPasswordConfirm: required: - new_password - confirm_new_password type: object properties: new_password: title: New password type: string minLength: 1 confirm_new_password: title: Confirm new password type: string minLength: 1 ResetPasswordRequest: required: - email type: object properties: email: title: Email type: string format: email minLength: 1 AuthToken: required: - username - password type: object properties: username: title: Username type: string minLength: 1 password: title: Password type: string minLength: 1 Login: required: - username - password type: object properties: recaptcha_token: title: Recaptcha token type: string username: title: Username type: string minLength: 1 password: title: Password type: string minLength: 1 UserProfile: required: - username - email - password type: object properties: username: title: Username type: string minLength: 1 email: title: Email type: string minLength: 1 password: title: Password type: string minLength: 1 is_superuser: title: Superuser status description: Designates that this user has all permissions without explicitly assigning them. type: boolean readOnly: true memory_quota: title: Memory quota type: integer readOnly: true User: required: - username - password type: object properties: id: title: ID type: integer readOnly: true email: title: Email address type: string format: email maxLength: 254 username: title: Username description: Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only. type: string pattern: ^[\w.@+-]+$ maxLength: 150 minLength: 1 password: title: Password type: string maxLength: 128 minLength: 1 recaptcha_token: title: Recaptcha token type: string ProjectDetail: description: Project where the activity was performed. type: object properties: pid: title: Pid description: A UUID identifying this project. type: string format: uuid readOnly: true name: title: Name description: Project name. type: string readOnly: true minLength: 1 Activity: description: Activity being notified. required: - user - project - description type: object properties: user: $ref: '#/definitions/UserDetail' project: $ref: '#/definitions/ProjectDetail' description: title: Description description: Activity description. type: string maxLength: 1000 minLength: 1 created: title: Created description: Date when the activity was performed. type: string format: date-time readOnly: true Notification: required: - activity type: object properties: nid: title: Nid description: A unique integer value identifying each notification type: integer readOnly: true activity: $ref: '#/definitions/Activity' seen: title: Seen description: Whether the notification was seen. type: boolean NotificationUpdate: type: object properties: nid: title: Nid description: A unique integer value identifying each notification type: integer readOnly: true seen: title: Seen description: Whether the notification was seen. type: boolean Permission: description: Users with permissions on this project. type: object properties: user: $ref: '#/definitions/UserDetail' permission: title: Permission description: Permission on this project. type: string enum: - OWNER - ADMIN - DEVELOPER - VIEWER SpiderJobEnvVar: description: Project env variables. required: - name - value type: object properties: evid: title: Evid description: A unique integer value identifying this job env variable. type: integer x-nullable: true name: title: Name description: Env variable name. type: string maxLength: 1000 minLength: 1 value: title: Value description: Env variable value. type: string maxLength: 12000 minLength: 1 masked: title: Masked description: Whether the env variable value is masked. type: boolean Project: required: - name type: object properties: pid: title: Pid description: A UUID identifying this project. type: string format: uuid readOnly: true name: title: Name description: Project's name. type: string maxLength: 1000 minLength: 1 category: title: Category description: Project's category. type: string enum: - NOT SPECIFIED - E-COMMERCE - LOGISTICS - FINANCE - EDUCATIONAL - TECHNOLOGY - OTHER_CATEGORY framework: title: Framework description: Project's framework. type: string enum: - SCRAPY - REQUESTS container_image: title: Container image description: Path of the project's container image. type: string readOnly: true minLength: 1 users: description: Users with permissions on this project. type: array items: $ref: '#/definitions/Permission' env_vars: description: Project env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 created: title: Created type: string readOnly: true last_modified: title: Last modified type: string readOnly: true ProjectUpdate: type: object properties: pid: title: Pid description: A UUID identifying this project. type: string format: uuid readOnly: true name: title: Name description: Project name. type: string minLength: 1 users: description: Affected users. type: array items: $ref: '#/definitions/UserDetail' email: title: Email description: Email address. type: string format: email minLength: 1 action: title: Action description: Performed action. type: string enum: - remove - add - update framework: title: Framework description: Set project framework. type: string enum: - SCRAPY - REQUESTS env_vars: description: Project env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' permission: title: Permission description: New permission. type: string enum: - ADMIN - DEVELOPER - VIEWER - OWNER data_status: title: Data status description: New data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: New data expiry days. type: integer apply_to_existing_spiders: title: Apply to existing spiders description: Apply data persistence settings to all existing spiders. type: boolean default: false ProjectActivity: required: - results - count type: object properties: results: description: Project activities. type: array items: $ref: '#/definitions/Activity' count: title: Count description: Project activities count. type: integer SpiderJobArg: description: Cron job arguments. required: - name - value type: object properties: name: title: Name description: Argument name. type: string maxLength: 1000 minLength: 1 value: title: Value description: Argument value. type: string maxLength: 1000 minLength: 1 SpiderJobTag: description: Cron job tags. required: - name type: object properties: name: title: Name description: Tag name. type: string maxLength: 50 minLength: 1 SpiderCronJob: description: Project Cronjobs. type: object properties: cjid: title: Cjid description: A unique integer value identifying this cron job. type: integer readOnly: true spider: title: Spider type: string readOnly: true created: title: Created description: Cron job creation date. type: string format: date-time readOnly: true name: title: Name description: Unique cron job name. type: string readOnly: true minLength: 1 cargs: description: Cron job arguments. type: array items: $ref: '#/definitions/SpiderJobArg' cenv_vars: description: Cron job env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' ctags: description: Cron job tags. type: array items: $ref: '#/definitions/SpiderJobTag' schedule: title: Schedule description: Cron job schedule definition. type: string maxLength: 20 status: title: Status description: Cron job status. type: string enum: - ACTIVE - DISABLED unique_collection: title: Unique collection description: True if this cron job stores its items in a unique collection. type: boolean data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 x-nullable: true resource_tier: title: Resource tier description: Resource tier for jobs created by this cron job. type: string enum: - TINY - XSMALL - SMALL - MEDIUM - LARGE - XLARGE - HUGE - XHUGE ProjectCronJob: required: - results - count type: object properties: results: description: Project Cronjobs. type: array items: $ref: '#/definitions/SpiderCronJob' count: title: Count description: Project cronjobs count. type: integer ProjectUsage: required: - network_usage - item_count - request_count - items_data_size - requests_data_size - logs_data_size type: object properties: pid: title: Pid type: string readOnly: true name: title: Name type: string readOnly: true network_usage: title: Network usage description: Amount of network bytes used. type: integer maximum: 18446744073709551615 minimum: 0 processing_time: title: Processing time type: string readOnly: true item_count: title: Item count description: Amount of items extracted. type: integer maximum: 18446744073709551615 minimum: 0 request_count: title: Request count description: Amount of requests made. type: integer maximum: 18446744073709551615 minimum: 0 items_data_size: title: Items data size description: Amount in bytes occupied by items in the database type: integer maximum: 18446744073709551615 minimum: 0 requests_data_size: title: Requests data size description: Amount in bytes occupied by requests in the database type: integer maximum: 18446744073709551615 minimum: 0 logs_data_size: title: Logs data size description: Amount in bytes occupied by logs in the database type: integer maximum: 18446744073709551615 minimum: 0 Deploy: required: - project - user type: object properties: did: title: Did description: A unique integer value identifying this deploy. type: integer readOnly: true project: title: Project description: Project UUID. type: string format: uuid user: $ref: '#/definitions/UserDetail' status: title: Status description: Deploy status. type: string enum: - SUCCESS - BUILDING - DOWNLOADING - FAILURE - CANCELED spiders_count: title: Spiders count description: Number of spiders in this deploy. type: string readOnly: true created: title: Created description: Deploy creation date. type: string format: date-time readOnly: true DeployCreate: type: object properties: did: title: Did description: A unique integer value identifying this deploy. type: integer readOnly: true status: title: Status description: Deploy status. type: string enum: - SUCCESS - BUILDING - DOWNLOADING - FAILURE - CANCELED created: title: Created description: Deploy creation date. type: string format: date-time readOnly: true project_zip: title: Project zip description: Zip file containing the project. type: string readOnly: true format: uri DeployUpdate: type: object properties: did: title: Did description: A unique integer value identifying this deploy. type: integer readOnly: true status: title: Status description: Deploy status. type: string enum: - SUCCESS - BUILDING - DOWNLOADING - FAILURE - CANCELED spiders_names: description: Spider names in this deploy. type: array items: type: string maxLength: 1000 minLength: 1 error_reason: title: Error reason description: Error logs to persist in deploy_logs (Mongo) on failure. type: string maxLength: 200000 x-nullable: true SpiderJob: description: Project jobs. type: object properties: jid: title: Jid description: A unique integer value identifying this job. type: integer readOnly: true spider: title: Spider type: string readOnly: true created: title: Created description: Job creation date. type: string format: date-time readOnly: true name: title: Name description: Unique job name. type: string readOnly: true minLength: 1 lifespan: title: Lifespan description: The elapsed seconds the spider job was running. type: number total_response_bytes: title: Total response bytes description: The total bytes received in responses. type: integer maximum: 18446744073709551615 minimum: 0 item_count: title: Item count description: The number of items extracted in the job. type: integer maximum: 18446744073709551615 minimum: 0 request_count: title: Request count description: The number of requests made by the spider job. type: integer maximum: 18446744073709551615 minimum: 0 args: description: Job arguments. type: array items: $ref: '#/definitions/SpiderJobArg' env_vars: description: Job env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' tags: description: Job tags. type: array items: $ref: '#/definitions/SpiderJobTag' job_status: title: Job status description: Current job status. type: string readOnly: true minLength: 1 cronjob: title: Cronjob description: Related cron job. type: integer x-nullable: true data_expiry_days: title: Data expiry days description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 x-nullable: true data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING - DELETED database_insertion_progress: title: Database insertion progress type: string readOnly: true storage_size: title: Storage size type: string readOnly: true resource_tier: title: Resource tier description: Resource tier for K8s pod allocation. type: string enum: - TINY - XSMALL - SMALL - MEDIUM - LARGE - XLARGE - HUGE - XHUGE peak_memory: title: Peak memory type: string readOnly: true ProjectJob: required: - results - count type: object properties: results: description: Project jobs. type: array items: $ref: '#/definitions/SpiderJob' count: title: Count description: Project jobs count. type: integer Spider: required: - name - project - data_status - data_expiry_days type: object properties: sid: title: Sid description: A unique integer value identifying this spider. type: integer readOnly: true name: title: Name description: Spider's name. type: string maxLength: 1000 minLength: 1 project: title: Project description: Project UUID. type: string format: uuid env_vars: description: Spider environment variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: Days before data expires. type: integer last_modified: title: Last modified description: Last time this spider was modified (latest deploy, job, or cronjob). type: string format: date-time readOnly: true SpiderUpdate: type: object properties: sid: title: Sid description: A UUID identifying this spider. type: string format: uuid readOnly: true env_vars: description: Project environment variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' data_status: title: Data status description: New data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: New data expiry days. type: integer SpiderCronJobCreate: required: - data_expiry_days - data_status type: object properties: cjid: title: Cjid description: A unique integer value identifying this cron job. type: integer readOnly: true name: title: Name description: Unique cron job name. type: string readOnly: true minLength: 1 cargs: description: Cron job arguments. type: array items: $ref: '#/definitions/SpiderJobArg' cenv_vars: description: Cron job env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' ctags: description: Cron job tags. type: array items: $ref: '#/definitions/SpiderJobTag' schedule: title: Schedule description: Cron job schedule definition. type: string maxLength: 20 unique_collection: title: Unique collection description: True if this cron job stores its items in a unique collection. type: boolean data_expiry_days: title: Data expiry days description: Days before data expires. type: integer data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING - DELETED resource_tier: title: Resource tier description: Resource tier for jobs created by this cron job. type: string enum: - TINY - XSMALL - SMALL - MEDIUM - LARGE - XLARGE - HUGE - XHUGE SpiderCronJobUpdate: type: object properties: cjid: title: Cjid description: A unique integer value identifying this cron job. type: integer readOnly: true status: title: Status description: Cron job status. type: string enum: - ACTIVE - DISABLED schedule: title: Schedule description: Cron job schedule definition. type: string maxLength: 20 unique_collection: title: Unique collection description: True if this cron job stores its items in a unique collection. type: boolean data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING data_expiry_days: title: Data expiry days description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 x-nullable: true resource_tier: title: Resource tier description: Resource tier for jobs created by this cron job. type: string enum: - TINY - XSMALL - SMALL - MEDIUM - LARGE - XLARGE - HUGE - XHUGE SpiderJobCreateEnvVar: description: Job env variables. required: - name - value type: object properties: evid: title: Evid description: Env var id. type: integer name: title: Name description: Env var name. type: string minLength: 1 value: title: Value description: Env var value. type: string minLength: 1 masked: title: Masked description: Env var masked. type: boolean default: false SpiderJobCreate: required: - data_status type: object properties: jid: title: Jid description: A unique integer value identifying this job. type: integer readOnly: true name: title: Name description: Unique job name. type: string readOnly: true minLength: 1 args: description: Job arguments. type: array items: $ref: '#/definitions/SpiderJobArg' env_vars: description: Job env variables. type: array items: $ref: '#/definitions/SpiderJobCreateEnvVar' tags: description: Job tags. type: array items: $ref: '#/definitions/SpiderJobTag' job_status: title: Job status description: Current job status. type: string readOnly: true minLength: 1 cronjob: title: Cronjob description: Related cron job. type: integer x-nullable: true data_expiry_days: title: Data expiry days description: Days before data expires. type: integer data_status: title: Data status description: Data status. type: string minLength: 1 resource_tier: title: Resource tier description: Resource tier for K8s pod allocation. type: string enum: - TINY - XSMALL - SMALL - MEDIUM - LARGE - XLARGE - HUGE - XHUGE SpiderJobUpdate: type: object properties: jid: title: Jid description: A unique integer value identifying this job. type: integer readOnly: true status: title: Status description: Job status. type: string enum: - IN_QUEUE - WAITING - RUNNING - STOPPED - COMPLETED - ERROR lifespan: title: Lifespan description: The elapsed seconds the spider job was running. type: number total_response_bytes: title: Total response bytes description: The total bytes received in responses. type: integer maximum: 18446744073709551615 minimum: 0 item_count: title: Item count description: The number of items extracted in the job. type: integer maximum: 18446744073709551615 minimum: 0 request_count: title: Request count description: The number of requests made by the spider job. type: integer maximum: 18446744073709551615 minimum: 0 data_status: title: Data status description: Job data status. type: string enum: - PERSISTENT - PENDING - DELETED data_expiry_days: title: Data expiry days description: Job data expiry days. type: integer proxy_usage_data: title: Proxy usage data description: Proxy Usage data. type: string error_reason: title: Error reason description: Error logs to persist in job_logs (Mongo) on failure. type: string maxLength: 200000 x-nullable: true UsageRecord: required: - processing_time - network_usage - item_count - request_count - items_data_size - requests_data_size - logs_data_size type: object properties: created_at: title: Created at description: Usage record creation date. type: string format: date-time readOnly: true processing_time: title: Processing time description: Time of CPU use. type: number network_usage: title: Network usage description: Amount of network bytes used. type: integer maximum: 18446744073709551615 minimum: 0 item_count: title: Item count description: Amount of items extracted. type: integer maximum: 18446744073709551615 minimum: 0 request_count: title: Request count description: Amount of requests made. type: integer maximum: 18446744073709551615 minimum: 0 items_data_size: title: Items data size description: Amount in bytes occupied by items in the database type: integer maximum: 18446744073709551615 minimum: 0 requests_data_size: title: Requests data size description: Amount in bytes occupied by requests in the database type: integer maximum: 18446744073709551615 minimum: 0 logs_data_size: title: Logs data size description: Amount in bytes occupied by logs in the database type: integer maximum: 18446744073709551615 minimum: 0 ProxyProvider: required: - name - description type: object properties: name: title: Name description: A name to identify the proxy type: string maxLength: 255 minLength: 1 description: title: Description description: A description for the proxy type: string maxLength: 1000 minLength: 1 proxyid: title: Proxyid description: A unique integer value identifying this proxy. type: integer readOnly: true ProxyProviderUpdate: required: - level - project_or_spider_id type: object properties: level: title: Level description: Spider or project type: string maxLength: 100 minLength: 1 project_or_spider_id: title: Project or spider id description: Project id where the update will be performed type: string maxLength: 100 minLength: 1 ProxyProviderResponse: required: - success type: object properties: success: title: Success type: boolean env_vars: description: Env vars for the instace(project, spider) type: array items: $ref: '#/definitions/SpiderJobEnvVar' JobsStats: type: object properties: total_jobs: title: Total jobs type: integer default: 0 waiting_jobs: title: Waiting jobs type: integer default: 0 running_jobs: title: Running jobs type: integer default: 0 stopped_jobs: title: Stopped jobs type: integer default: 0 completed_jobs: title: Completed jobs type: integer default: 0 in_queue_jobs: title: In queue jobs type: integer default: 0 error_jobs: title: Error jobs type: integer default: 0 PagesStats: type: object properties: total_pages: title: Total pages type: integer default: 0 scraped_pages: title: Scraped pages type: integer default: 0 missed_pages: title: Missed pages type: integer default: 0 StatusCodesStats: type: object properties: status_200: title: Status 200 type: integer default: 0 status_301: title: Status 301 type: integer default: 0 status_302: title: Status 302 type: integer default: 0 status_401: title: Status 401 type: integer default: 0 status_403: title: Status 403 type: integer default: 0 status_404: title: Status 404 type: integer default: 0 status_429: title: Status 429 type: integer default: 0 status_500: title: Status 500 type: integer default: 0 LogsStats: type: object properties: total_logs: title: Total logs type: integer default: 0 debug_logs: title: Debug logs type: integer default: 0 info_logs: title: Info logs type: integer default: 0 warning_logs: title: Warning logs type: integer default: 0 error_logs: title: Error logs type: integer default: 0 critical_logs: title: Critical logs type: integer default: 0 FieldCoverageStats: type: object properties: field_name: title: Field name type: string default: '' minLength: 1 field_count: title: Field count type: integer default: 0 field_coverage: title: Field coverage type: number default: 0.0 CoverageStats: type: object properties: total_items: title: Total items type: integer default: 0 total_items_coverage: title: Total items coverage type: number default: 0.0 fields: type: array items: $ref: '#/definitions/FieldCoverageStats' Stats: required: - pages - status_codes - logs type: object properties: jobs: $ref: '#/definitions/JobsStats' pages: $ref: '#/definitions/PagesStats' items_count: title: Items count type: integer default: 0 runtime: title: Runtime type: number default: 00:00:00 status_codes: $ref: '#/definitions/StatusCodesStats' success_rate: title: Success rate type: number default: 0.0 logs: $ref: '#/definitions/LogsStats' coverage: $ref: '#/definitions/CoverageStats' ProjectStats: required: - date - stats type: object properties: date: title: Date type: string format: date stats: $ref: '#/definitions/Stats' SpidersStats: required: - date - stats type: object properties: date: title: Date type: string format: date stats: $ref: '#/definitions/Stats' SpiderJobStats: required: - stats type: object properties: jid: title: Jid description: A unique integer value identifying this job. type: integer readOnly: true spider: title: Spider type: string readOnly: true created: title: Created description: Job creation date. type: string format: date-time readOnly: true name: title: Name description: Unique job name. type: string readOnly: true minLength: 1 lifespan: title: Lifespan description: The elapsed seconds the spider job was running. type: number total_response_bytes: title: Total response bytes description: The total bytes received in responses. type: integer maximum: 18446744073709551615 minimum: 0 item_count: title: Item count description: The number of items extracted in the job. type: integer maximum: 18446744073709551615 minimum: 0 request_count: title: Request count description: The number of requests made by the spider job. type: integer maximum: 18446744073709551615 minimum: 0 args: description: Job arguments. type: array items: $ref: '#/definitions/SpiderJobArg' env_vars: description: Job env variables. type: array items: $ref: '#/definitions/SpiderJobEnvVar' tags: description: Job tags. type: array items: $ref: '#/definitions/SpiderJobTag' job_status: title: Job status description: Current job status. type: string readOnly: true minLength: 1 cronjob: title: Cronjob description: Related cron job. type: integer x-nullable: true data_expiry_days: title: Data expiry days description: Days before data is deleted. type: integer maximum: 65535 minimum: 0 x-nullable: true data_status: title: Data status description: Data status. type: string enum: - PERSISTENT - PENDING - DELETED stats: $ref: '#/definitions/Stats' JobsPagination: required: - count - results type: object properties: count: title: Count type: integer next: title: Next type: string format: uri readOnly: true x-nullable: true previous: title: Previous type: string format: uri readOnly: true x-nullable: true results: type: array items: $ref: '#/definitions/SpiderJobStats' SpidersPagination: required: - count - results type: object properties: count: title: Count type: integer next: title: Next type: string format: uri readOnly: true x-nullable: true previous: title: Previous type: string format: uri readOnly: true x-nullable: true results: type: array items: $ref: '#/definitions/Spider' MeteringReport: required: - project_id - resource_kind - resource_id - interval_start - interval_end - metrics - idempotency_key type: object properties: project_id: title: Project id description: Project UUID (``Project.pid``) this usage rolls up to. type: string format: uuid resource_kind: title: Resource kind type: string maxLength: 64 minLength: 1 resource_id: title: Resource id type: string maxLength: 512 minLength: 1 interval_start: title: Interval start type: string format: date-time interval_end: title: Interval end type: string format: date-time metrics: title: Metrics description: "Open metric payload (network_bytes, storage_bytes, \u2026)." type: object additionalProperties: type: object idempotency_key: title: Idempotency key type: string maxLength: 512 minLength: 1 kind: title: Kind type: string enum: - JOB_CLOSE - DATA_DELETE - ADJUSTMENT - DELTA_SLICE default: DELTA_SLICE source_ref: title: Source ref type: string default: '' maxLength: 512 reporter: title: Reporter description: Emitting service (e.g. bitmaker-proxy). Defaults to estela. type: string default: '' maxLength: 64 adjustment_reason: title: Adjustment reason type: string enum: - RECONCILE_SCRAPY_FINAL - RECONCILE_STORAGE - MANUAL default: '' MeteringReportResponse: required: - id - recorded_at - duplicate - resource_kind - resource_id type: object properties: id: title: Id type: string format: uuid recorded_at: title: Recorded at type: string format: date-time duplicate: title: Duplicate type: boolean resource_kind: title: Resource kind type: string minLength: 1 resource_id: title: Resource id type: string minLength: 1