zwf
462 lines
17 KiB
Python
462 lines
17 KiB
Python
import random
|
||
from typing import Union, Optional, Callable
|
||
|
||
import pandas as pd
|
||
from sqlalchemy import select, delete
|
||
from tornado_swagger.model import register_swagger_model
|
||
from wtforms import StringField, IntegerField
|
||
from wtforms.validators import Length
|
||
|
||
import models
|
||
from models.common_model import CommonModel
|
||
from models.db_models import TD3iDcmTaskFileUpload
|
||
from paste.core.logging import echo_log
|
||
from paste.util.pagination import Pagination
|
||
from paste.web.form import ModelForm
|
||
|
||
|
||
class DcmTaskFileUploadForm(ModelForm):
|
||
"""
|
||
文件上传关联表单验证类（完全映射 TD3iDcmTaskFileUpload 字段）。
|
||
|
||
用于验证和处理文件上传关联数据的表单。
|
||
字段完全映射数据库表 t_d3i_dcm_task_file_upload 的字段结构。
|
||
"""
|
||
|
||
# 关联信息
|
||
dcm_task_id = IntegerField('任务ID', validators=[Length(min=1, message='任务ID不能为空')])
|
||
dcm_task_attachment_id = IntegerField('附件ID', validators=[Length(min=1, message='附件ID不能为空')])
|
||
dcm_media_id = IntegerField('数字城管附件ID', validators=[Length(min=1, message='数字城管附件ID不能为空')])
|
||
oa_media_id = IntegerField('OA附件ID', validators=[Length(min=1, message='OA附件ID不能为空')])
|
||
|
||
# 文件信息
|
||
file_hash = StringField('文件哈希值', validators=[Length(max=256, message='文件哈希值长度不能超过256字符')])
|
||
|
||
# 状态与时间
|
||
status = IntegerField('上传状态', validators=[Length(min=0, max=1, message='状态只能是0或1')])
|
||
created_at = IntegerField('创建时间戳（毫秒）', validators=[Length(min=1, message='创建时间不能为空')])
|
||
created_by = StringField('创建者', validators=[Length(max=64, message='创建者长度不能超过64字符')])
|
||
|
||
def process(self, formdata=None, obj=None, **kwargs):
|
||
if formdata:
|
||
for name, values in formdata.items():
|
||
if isinstance(values, list) and values:
|
||
formdata[name] = [v.strip() if isinstance(v, str) else v for v in values]
|
||
elif isinstance(values, str):
|
||
formdata[name] = values.strip()
|
||
super().process(formdata, obj, **kwargs)
|
||
|
||
|
||
class DcmTaskFileUploadBase(TD3iDcmTaskFileUpload, CommonModel):
|
||
"""
|
||
文件上传关联基础类（完全映射 TD3iDcmTaskFileUpload 字段）。
|
||
|
||
封装所有与文件上传关联相关的通用操作方法。
|
||
"""
|
||
|
||
FieldMapping = {
|
||
'dcm_task_id': 'dcmTaskId',
|
||
'dcm_task_attachment_id': 'dcmTaskAttachmentId',
|
||
'dcm_media_id': 'dcmMediaId',
|
||
'oa_media_id': 'oaMediaId',
|
||
'file_hash': 'fileHash',
|
||
'status': 'status',
|
||
'created_at': 'createdAt',
|
||
'created_by': 'createdBy',
|
||
}
|
||
"""
|
||
文件上传关联数据映射
|
||
"""
|
||
|
||
@classmethod
|
||
async def is_exist(cls, dcm_task_id: Union[str, int], dcm_task_attachment_id: Union[str, int]):
|
||
"""
|
||
检查是否已存在相同任务ID和附件ID的记录。
|
||
"""
|
||
_query = select(cls).where(
|
||
cls.dcm_task_id == dcm_task_id, cls.dcm_task_attachment_id == dcm_task_attachment_id
|
||
)
|
||
_record: cls = await cls.query_first(_query)
|
||
return _record
|
||
|
||
@classmethod
|
||
async def find_by_ids(cls, ids: list[Union[str, int]]):
|
||
"""
|
||
根据ID列表批量查找记录。
|
||
"""
|
||
_query = select(cls).where(cls.id.in_(ids))
|
||
_records: list[cls] = (await cls.orm_execute_scalars(_query)).all()
|
||
return _records
|
||
|
||
@classmethod
|
||
async def search_base(cls, is_paging=True, **kwargs):
|
||
"""
|
||
按参数搜索文件上传记录的基础方法。
|
||
|
||
支持字段：
|
||
- dcm_task_id, dcm_task_attachment_id, dcm_media_id, oa_media_id, status
|
||
|
||
:param is_paging: 是否分页
|
||
:param kwargs: 查询参数
|
||
:key int page_number: 页码（缺省随机1~100）
|
||
:key int page_size: 每页数量（缺省20）
|
||
:key dict sort_clause: 排序配置，如 {'created_at': 'asc'}
|
||
:key int dcm_task_id: 精确匹配任务ID
|
||
:key int dcm_task_attachment_id: 精确匹配附件ID
|
||
:key int dcm_media_id: 精确匹配数字城管附件ID
|
||
:key int oa_media_id: 精确匹配OA附件ID
|
||
:key int status: 精确匹配上传状态
|
||
:return: (DataFrame, Pagination)
|
||
"""
|
||
page_number = kwargs.get('page_number', random.randint(1, 100))
|
||
page_size = kwargs.get('page_size', 20)
|
||
kwargs.update({'page_number': page_number, 'page_size': page_size})
|
||
|
||
_query = select(cls).where(
|
||
*cls.search_wheres(**kwargs)
|
||
)
|
||
|
||
_paging = None
|
||
if is_paging:
|
||
_row_count = await cls.query_count(_query)
|
||
_paging = Pagination(_row_count).paging(page_number, page_size)
|
||
_data_query = _query.limit(page_size).offset(_paging.offset_size)
|
||
else:
|
||
_data_query = _query.where()
|
||
|
||
_sort_clause = cls.sort_clauses(kwargs.get('sort_clause', {}))
|
||
if _sort_clause:
|
||
_data_query = _data_query.order_by(*_sort_clause)
|
||
else:
|
||
_data_query = _data_query.order_by(cls.created_at)
|
||
|
||
_df = await cls.query_as_df(_data_query)
|
||
if not _df.empty:
|
||
_df[cls.id.key] = _df[cls.id.key].astype(str)
|
||
_df[cls.dcm_task_id.key] = _df[cls.dcm_task_id.key].astype(str)
|
||
_df[cls.dcm_task_attachment_id.key] = _df[cls.dcm_task_attachment_id.key].astype(str)
|
||
_df.replace(models.EmptyInDF + models.EmptyDatetimeInDF, '', inplace=True)
|
||
|
||
return _df, _paging
|
||
|
||
@classmethod
|
||
async def search_by_attachment_ids(cls, attachment_ids: list[Union[str, int]]):
|
||
query = select(
|
||
cls.dcm_task_attachment_id, cls.oa_media_id, cls.updated_at
|
||
).where(
|
||
cls.dcm_task_attachment_id.in_(attachment_ids)
|
||
)
|
||
return await cls.query_as_df(query)
|
||
|
||
@classmethod
|
||
async def search(cls, **kwargs):
|
||
"""
|
||
按参数搜索文件上传记录，返回分页格式数据。
|
||
"""
|
||
_df, _paging = await cls.search_base(**kwargs)
|
||
return {
|
||
'total': _paging.row_count if _paging else len(_df),
|
||
'rows': _df.to_dict('records'),
|
||
'pagination': {
|
||
'page_number': _paging.page_number if _paging else 1,
|
||
'page_count': _paging.page_count if _paging else 1,
|
||
'page_size': _paging.page_size if _paging else 20,
|
||
},
|
||
}
|
||
|
||
@classmethod
|
||
async def exists_relation(cls, data_df: pd.DataFrame):
|
||
"""
|
||
查找 data_df 中在数据库中已存在和不存在的记录。根据 dcm_task_id + dcm_task_attachment_id 判断。
|
||
|
||
:param data_df: 输入的数据框架，必须包含 dcm_task_id 和 dcm_task_attachment_id 列
|
||
:return: (exists_df: pd.DataFrame, latest_df: pd.DataFrame)
|
||
"""
|
||
if data_df.empty:
|
||
return pd.DataFrame(), pd.DataFrame()
|
||
|
||
pairs = data_df[[cls.dcm_task_id.key, cls.dcm_task_attachment_id.key]].drop_duplicates().values.tolist()
|
||
if not pairs:
|
||
return pd.DataFrame(), data_df.copy()
|
||
|
||
_query = select(cls.id, cls.dcm_task_id, cls.dcm_task_attachment_id).where(
|
||
(cls.dcm_task_id.in_([p[0] for p in pairs])) &
|
||
(cls.dcm_task_attachment_id.in_([p[1] for p in pairs]))
|
||
)
|
||
exists_df = await cls.query_as_df(_query)
|
||
|
||
if exists_df.empty:
|
||
return pd.DataFrame(), data_df.copy()
|
||
|
||
key_to_id_map = dict(zip(zip(exists_df[cls.dcm_task_id.key], exists_df[cls.dcm_task_attachment_id.key]), exists_df[cls.id.key]))
|
||
|
||
mask_exists = data_df.apply(lambda row: (row[cls.dcm_task_id.key], row[cls.dcm_task_attachment_id.key]) in key_to_id_map, axis=1)
|
||
exists_df = data_df[mask_exists].copy()
|
||
exists_df[cls.id.key] = exists_df.apply(lambda row: key_to_id_map[(row[cls.dcm_task_id.key], row[cls.dcm_task_attachment_id.key])], axis=1)
|
||
latest_df = data_df[~mask_exists].copy()
|
||
|
||
return exists_df, latest_df
|
||
|
||
@classmethod
|
||
async def fill_file_upload(cls, data_df: pd.DataFrame, index_field: str = 'dcm_task_attachment_id',
|
||
column_name: str = 'file_uploads',
|
||
preprocessing: Optional[Callable] = None):
|
||
"""
|
||
填充文件上传数据到数据框架。
|
||
|
||
:param pandas.DataFrame data_df: 待填充的数据框架
|
||
:param str index_field: 索引字段，一般是任务ID
|
||
:param str column_name: 填充时新增列名称，默认为`file_uploads`
|
||
:param preprocessing: 预处理函数
|
||
:return: 填充后的数据框架
|
||
"""
|
||
if data_df.empty:
|
||
return pd.DataFrame()
|
||
|
||
_task_attachment_ids = list(set(data_df[index_field].unique().tolist()))
|
||
if not _task_attachment_ids:
|
||
return pd.DataFrame()
|
||
|
||
_query = select(cls).where(cls.dcm_task_attachment_id.in_(_task_attachment_ids))
|
||
_upload_df: pd.DataFrame = await cls.query_as_df(_query)
|
||
if not _upload_df.empty:
|
||
_upload_df.replace(models.EmptyInDF+models.EmptyDatetimeInDF, '', inplace=True)
|
||
_upload_df[cls.id.key] = _upload_df[cls.id.key].astype(str)
|
||
_upload_df[cls.dcm_task_id.key] = _upload_df[cls.dcm_task_id.key].astype(str)
|
||
_upload_df[cls.dcm_task_attachment_id.key] = _upload_df[cls.dcm_task_attachment_id.key].astype(str)
|
||
|
||
_upload_df['index_id'] = _upload_df[cls.id.key]
|
||
_upload_df.set_index(['index_id'], inplace=True)
|
||
|
||
if isinstance(preprocessing, Callable):
|
||
_upload_df = preprocessing(_upload_df)
|
||
|
||
data_df[column_name] = data_df[index_field].apply(
|
||
lambda x: next(iter(_upload_df.query(f"{cls.dcm_task_attachment_id.key}=='{x}'").to_dict('records')), {})
|
||
)
|
||
else:
|
||
data_df[column_name] = [[] for _ in range(len(data_df))]
|
||
|
||
return _upload_df
|
||
|
||
|
||
@register_swagger_model
|
||
class DcmTaskFileUpload(DcmTaskFileUploadBase):
|
||
"""
|
||
文件上传关联业务模型类（主业务类，完全继承 TD3iDcmTaskFileUpload 字段）。
|
||
|
||
---
|
||
description: 文件上传关联表
|
||
type: object
|
||
properties:
|
||
id:
|
||
description: 主键ID
|
||
type: integer
|
||
example: 1001
|
||
readOnly: true
|
||
dcm_task_id:
|
||
description: 任务唯一标志
|
||
type: integer
|
||
example: 2001
|
||
dcm_task_attachment_id:
|
||
description: 附件ID（数字城管附件）
|
||
type: integer
|
||
example: 3001
|
||
dcm_media_id:
|
||
description: 附件ID（数字城管媒体）
|
||
type: integer
|
||
example: 3002
|
||
oa_media_id:
|
||
description: 附件ID（OA媒体）
|
||
type: integer
|
||
example: 3003
|
||
file_hash:
|
||
description: 文件哈希值
|
||
type: string
|
||
example: "a1b2c3d4e5..."
|
||
maxLength: 256
|
||
status:
|
||
description: 上传状态（0:未上传/失败，1:成功）
|
||
type: integer
|
||
example: 1
|
||
created_at:
|
||
description: 创建时间，ISO格式的日期时间字符串
|
||
type: string
|
||
format: date-time
|
||
example: "2024-01-15 10:30:00"
|
||
readOnly: true
|
||
created_by:
|
||
description: 创建者用户名
|
||
type: string
|
||
example: "D3I"
|
||
readOnly: true
|
||
updated_at:
|
||
description: 修改时间，ISO格式的日期时间字符串
|
||
type: string
|
||
format: date-time
|
||
example: "2024-01-16 14:25:00"
|
||
readOnly: true
|
||
updated_by:
|
||
description: 修改者用户名
|
||
type: string
|
||
example: "editor"
|
||
readOnly: true
|
||
"""
|
||
|
||
@classmethod
|
||
async def create(cls, **kwargs):
|
||
"""
|
||
创建新的文件上传关联记录。
|
||
|
||
业务流程：
|
||
1. 使用 DcmTaskFileUploadForm 验证表单数据
|
||
2. 检查是否已存在相同任务ID和附件ID的记录
|
||
3. 创建新对象，设置创建者
|
||
4. 保存到数据库
|
||
5. 返回创建对象
|
||
|
||
:param kwargs: 附件参数字典
|
||
:return: 新建的文件上传记录
|
||
:raises AssertionError: 当记录已存在时抛出
|
||
:raises ValidationError: 当表单验证失败时抛出
|
||
"""
|
||
for _k, _v in kwargs.items():
|
||
if isinstance(_v, str):
|
||
kwargs[_k] = _v.strip()
|
||
|
||
_form = DcmTaskFileUploadForm(formdata=kwargs)
|
||
_form.validate_form()
|
||
|
||
# 检查是否已存在
|
||
_exist = await cls.is_exist(_form.dcm_task_id.data, _form.dcm_task_attachment_id.data)
|
||
assert _exist is None, "相同任务ID和附件ID的文件上传记录已存在，不能重复创建。"
|
||
|
||
_record = cls().copy_from_dict(_form.data, skip_none=True).before_save()
|
||
await _record.async_save()
|
||
return _record
|
||
|
||
@classmethod
|
||
async def delete(cls, id: Union[str, int]):
|
||
"""
|
||
软删除（不推荐物理删除，但可设 status=-1 或删除记录）
|
||
|
||
本系统暂不支持软删除，建议直接物理删除。
|
||
"""
|
||
_record: cls = await cls.async_find_by_id(id)
|
||
assert _record, f"根据 ID {id} 未找到文件上传记录。"
|
||
|
||
_del_query = delete(cls).where(cls.id == _record.id)
|
||
_del_count = (await cls.raw_execute(_del_query)).rowcount
|
||
echo_log(f'已删除文件上传记录（ID：{_record.id}）.')
|
||
return _record
|
||
|
||
@classmethod
|
||
async def modify(cls, id: Union[str, int], **kwargs):
|
||
"""
|
||
修改文件上传记录。
|
||
|
||
注意：仅允许修改 status、created_by 等非核心关联字段。
|
||
核心字段（dcm_task_id, dcm_task_attachment_id, dcm_media_id, oa_media_id）不允许修改。
|
||
|
||
:param id: 记录ID
|
||
:param kwargs: 更新字段
|
||
:return: 更新后的记录
|
||
:raises AssertionError: 当记录不存在或字段被非法修改时抛出
|
||
:raises ValidationError: 当表单验证失败时抛出
|
||
"""
|
||
for _k, _v in kwargs.items():
|
||
if isinstance(_v, str):
|
||
kwargs[_k] = _v.strip()
|
||
|
||
_form = DcmTaskFileUploadForm(formdata=kwargs)
|
||
_form.validate_form()
|
||
|
||
# 核心字段禁止修改
|
||
protected_fields = {'dcm_task_id', 'dcm_task_attachment_id', 'dcm_media_id', 'oa_media_id'}
|
||
disallowed_fields = set(kwargs.keys()) & protected_fields
|
||
assert not disallowed_fields, f"禁止修改核心字段: {disallowed_fields}"
|
||
|
||
_record: cls = await cls.async_find_by_id(id)
|
||
assert _record, f'查无此文件上传记录（ID：{id}）。'
|
||
|
||
# 允许更新的字段
|
||
allowed_fields = {'status', 'created_by'}
|
||
update_data = {k: v for k, v in _form.data.items() if k in allowed_fields and v is not None}
|
||
|
||
_record.copy_from_dict(update_data, skip_none=True).before_save()
|
||
await _record.async_save()
|
||
return _record
|
||
|
||
@classmethod
|
||
async def create_batch(cls, data_df: pd.DataFrame):
|
||
"""
|
||
批量创建文件上传记录（传入数据应为全新记录）。
|
||
|
||
:param data_df: 包含字段 dcm_task_id, dcm_task_attachment_id, dcm_media_id, oa_media_id, file_hash, status, created_at, created_by 的 DataFrame
|
||
:return: 成功创建的数量
|
||
"""
|
||
if data_df.empty:
|
||
return 0
|
||
|
||
records = data_df.to_dict('records')
|
||
records = [cls().copy_from_dict(r, skip_none=True).before_save() for r in records]
|
||
|
||
session = cls.get_aio_session()
|
||
try:
|
||
session.add_all(records)
|
||
await session.commit()
|
||
except Exception as e:
|
||
await session.rollback()
|
||
raise e
|
||
finally:
|
||
await session.close()
|
||
echo_log(f"批量创建成功：创建 {len(records)} 条文件上传记录。")
|
||
return len(records)
|
||
|
||
@classmethod
|
||
async def modify_batch(cls, data_df: pd.DataFrame):
|
||
"""
|
||
批量修改文件上传记录。
|
||
|
||
:param data_df: 必须包含 id 列的 DataFrame
|
||
:return: 成功更新的数量
|
||
"""
|
||
if data_df.empty:
|
||
return 0
|
||
|
||
if 'id' not in data_df.columns:
|
||
echo_log("错误：modify_batch 要求输入数据必须包含 'id' 列（主键）")
|
||
return 0
|
||
|
||
update_data = data_df.to_dict('records')
|
||
|
||
session = cls.get_aio_session()
|
||
try:
|
||
await session.run_sync(
|
||
lambda sync_session: sync_session.bulk_update_mappings(cls, update_data)
|
||
)
|
||
await session.commit()
|
||
updated_count = len(update_data)
|
||
except Exception as e:
|
||
await session.rollback()
|
||
raise e
|
||
finally:
|
||
await session.close()
|
||
|
||
echo_log(f"批量修改成功：更新 {updated_count} 条文件上传记录。")
|
||
return updated_count
|
||
|
||
@classmethod
|
||
async def save_batch(cls, data_df: pd.DataFrame):
|
||
"""
|
||
批量保存数据，自动处理新建和更新。
|
||
|
||
:param data_df: 要保存的数据框架，必须包含 dcm_task_id 和 dcm_task_attachment_id
|
||
:return: (新建数量, 更新数量)
|
||
"""
|
||
_exists_df, _latest_df = await DcmTaskFileUpload.exists_relation(data_df)
|
||
|
||
_created_count = await DcmTaskFileUpload.create_batch(_latest_df)
|
||
_updated_count = await DcmTaskFileUpload.modify_batch(_exists_df)
|
||
|
||
return _created_count, _updated_count |