Haokai/d3i-szct
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

恢复项目

Browse Source
This commit is contained in:
zwf
2026-06-03 09:58:49 +08:00
parent dd72f9008f
commit 85ec8bf063
20 changed files with 305 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dock/govc/govc_scrape.py
+96 -10
View File
@@ -7,7 +7,9 @@ from typing import Optional, Union
from sqlalchemy import select, desc
import dock
from dock.govc import govc_scrape_dept_feedback, govc_scrape_return_visit, govc_scrape_finish_info, govc_scrape_order
from dock.govc import govc_scrape_dept_feedback, govc_scrape_return_visit, govc_scrape_finish_info, govc_scrape_detail, \
govc_scrape_process, govc_scrape_requester, govc_scrape_delay_info, govc_scrape_contact_info, \
govc_scrape_order_status, govc_scrape_history_order, govc_scrape_supervision, govc_scrape_order
from models.govc_task import GovcTask
from paste.core.logging import echo_log
from paste.web import requests
@@ -52,8 +54,16 @@ async def fetch_govc_task(fetch_size: int = 60, task_id: Optional[Union[str, int
# 构建请求队列
feedback_queue = asyncio.Queue()
result_info_queue = asyncio.Queue()
finish_info_queue = asyncio.Queue()
contact_info_queue = asyncio.Queue()
delay_info_queue = asyncio.Queue()
detail_queue = asyncio.Queue()
status_queue = asyncio.Queue()
process_queue = asyncio.Queue()
requester_queue = asyncio.Queue()
return_visit_queue = asyncio.Queue()
history_order_queue = asyncio.Queue()
supervision_queue = asyncio.Queue()
# 向队列中填充请求对象
echo_log(f"正在准备请求队列...")
for _h, _row in task_df.iterrows():
@@ -61,15 +71,59 @@ async def fetch_govc_task(fetch_size: int = 60, task_id: Optional[Union[str, int
_row.get(GovcTask.c_guid.key))
setattr(_feedback_request, 'task_id', _row.get(GovcTask.id.key))
await feedback_queue.put(_feedback_request)
_result_info_request = await govc_scrape_return_visit.get_return_visit_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_result_info_request, 'task_id', _row.get(GovcTask.id.key))
await result_info_queue.put(_result_info_request)
_finish_info_request = await govc_scrape_finish_info.get_finish_info_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_finish_info_request, 'task_id', _row.get(GovcTask.id.key))
await finish_info_queue.put(_finish_info_request)
_contact_info_request = await govc_scrape_contact_info.get_contact_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_contact_info_request, 'task_id', _row.get(GovcTask.id.key))
await contact_info_queue.put(_contact_info_request)
_delay_info_request = await govc_scrape_delay_info.get_delay_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_delay_info_request, 'task_id', _row.get(GovcTask.id.key))
await delay_info_queue.put(_delay_info_request)
_detail_request = await govc_scrape_detail.get_detail_request(_row.get(GovcTask.c_guid.key))
setattr(_detail_request, 'task_id', _row.get(GovcTask.id.key))
await detail_queue.put(_detail_request)
_status_request = await govc_scrape_order_status.get_fetch_status_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_status_request, 'task_id', _row.get(GovcTask.id.key))
await status_queue.put(_status_request)
_process_request = await govc_scrape_process.get_process_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_process_request, 'task_id', _row.get(GovcTask.id.key))
await process_queue.put(_process_request)
_requester_request = await govc_scrape_requester.get_requster_request(_row.get(GovcTask.c_guid.key))
setattr(_requester_request, 'task_id', _row.get(GovcTask.id.key))
await requester_queue.put(_requester_request)
_return_visit_request = await govc_scrape_return_visit.get_return_visit_request(_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key))
setattr(_return_visit_request, 'task_id', _row.get(GovcTask.id.key))
await return_visit_queue.put(_return_visit_request)
_history_order_request = await govc_scrape_history_order.get_history_order_request(
_row.get(GovcTask.c_guid.key))
setattr(_history_order_request, 'task_id', _row.get(GovcTask.id.key))
await history_order_queue.put(_history_order_request)
for supervise_type in ('zx', 'bm', 'bmhj'):
_supervision_request = await govc_scrape_supervision.get_supervision_request(
_row.get(GovcTask.pvi_guid.key),
_row.get(GovcTask.c_guid.key),
supervise_type)
setattr(_supervision_request, 'task_id', _row.get(GovcTask.id.key))
setattr(_supervision_request, 'supervise_type', supervise_type)
await supervision_queue.put(_supervision_request)
echo_log(f"抓取待办详细数据...")
tasks = [
requests.async_concurrency(
@@ -77,12 +131,44 @@ async def fetch_govc_task(fetch_size: int = 60, task_id: Optional[Union[str, int
after_request=govc_scrape_dept_feedback.after_feedback_request
),
requests.async_concurrency(
result_info_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_result_info.after_result_info_request
finish_info_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_finish_info.after_finish_info_request),
requests.async_concurrency(
contact_info_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_contact_info.after_contact_request
),
requests.async_concurrency(
finish_info_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_finish_info.after_finish_info_request)
delay_info_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_delay_info.after_delay_request
),
requests.async_concurrency(
detail_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_detail.after_detail_request
),
requests.async_concurrency(
status_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_order_status.after_fetch_status_request
),
requests.async_concurrency(
process_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_process.after_process_request
),
requests.async_concurrency(
request_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_requester.after_requester_request
),
requests.async_concurrency(
return_visit_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_return_visit.after_return_visit_request
),
requests.async_concurrency(
history_order_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_history_order.after_history_order_request
),
requests.async_concurrency(
supervision_queue, con_count=dock.CONCURRENCY_COUNT, retry=dock.MAX_RETRY_COUNT,
after_request=govc_scrape_supervision.after_return_visit_request
)
]
await asyncio.gather(*tasks)