首次提交

This commit is contained in:
zwf
2026-06-02 16:26:10 +08:00
commit 291e6fcaae
79 changed files with 11283 additions and 0 deletions
View File
+168
View File
@@ -0,0 +1,168 @@
import atexit
import os
import signal
import sys
from typing import Callable
import psutil
from paste.util import ufile
class DaemonizeService:
"""
驻内存服务。
"""
def __init__(self, pid_file, name: str = '', stdin: str = None, stdout: str = None, stderr: str = None):
"""
初始化服务。
:param pid_file: pid 文件路径
:param name: 服务名称
:param stdin: 输入文件路径
:param stdout: 输出文件路径
:param stderr: 错误日志文件路径
"""
self.start_callback = None
self.start_callback_args = ()
self.start_callback_kwargs = {}
self.term_callback = None
self.term_callback_args = ()
self.term_callback_kwargs = {}
self.pid_file = pid_file
self.name = name
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
def set_start_callback(self, callback, *args, **kwargs):
"""
设置启动回调函数。不返回参数。
:param callback: 回调函数
:param args: 回调参数
:param kwargs: 回调参数
"""
self.start_callback = callback
self.start_callback_args = args
self.start_callback_kwargs = kwargs
def set_term_callback(self, callback, *args, **kwargs):
"""
设置终止回调函数。不返回参数。
:param callback: 回调函数
:param args: 回调参数
:param kwargs: 回调参数
"""
self.term_callback = callback
self.term_callback_args = args
self.term_callback_kwargs = kwargs
def daemonize(self):
"""
设置和启动常驻服务程序。
"""
if os.path.exists(self.pid_file):
_pid = int(ufile.read_to_buffer(self.pid_file).decode('utf8').strip())
if psutil.pid_exists(_pid):
raise RuntimeError(f"[{self.name}] 正在运行.")
else:
os.remove(self.pid_file)
try:
if os.fork() > 0:
raise SystemExit(0) # Parent exit
except OSError:
raise RuntimeError('创建子进程失败 #1.')
os.chdir(os.path.abspath(os.path.curdir))
os.umask(0)
os.setsid()
try:
if os.fork() > 0:
raise SystemExit(0)
except OSError:
raise RuntimeError('创建子进程失败 #2.')
sys.stdout.flush()
sys.stderr.flush()
# 替换 stdin, stdout, 和 stderr 的文件描述符
if self.stdin is not None:
with open(self.stdin, 'rb', 0) as file:
os.dup2(file.fileno(), sys.stdin.fileno())
if self.stdout is not None:
with open(self.stdout, 'ab', 0) as file:
os.dup2(file.fileno(), sys.stdout.fileno())
if self.stderr is not None:
with open(self.stderr, 'ab', 0) as file:
os.dup2(file.fileno(), sys.stderr.fileno())
# 写入 PID 文件
with open(self.pid_file, 'w') as file:
print(os.getpid(), file=file)
# 注册退出函数,进程退出时(包括异常退出)移除pid文件
atexit.register(lambda: os.remove(self.pid_file))
# 监听终止信号,绑定到处理程序
signal.signal(signal.SIGTERM, self.sigterm_handler)
def sigterm_handler(self, signum, frame):
"""
终止信号处理程序。这里是执行回调函数,处理服务退出前的准备。
:param signum: 信号代码
:param frame: 帧
:return: 程序退出码
"""
if isinstance(self.term_callback, Callable):
self.term_callback(*self.term_callback_args, **self.term_callback_kwargs)
raise SystemExit(1)
def start(self):
"""
启动服务。
"""
try:
self.daemonize()
if isinstance(self.start_callback, Callable):
self.start_callback(*self.start_callback_args, **self.start_callback_kwargs)
except RuntimeError as e:
print(e, file=sys.stderr)
raise SystemExit(1)
def stop(self):
"""
停止服务。
"""
if os.path.exists(self.pid_file):
_pid = int(ufile.read_to_buffer(self.pid_file).decode('utf8').strip())
if psutil.pid_exists(_pid):
os.kill(_pid, signal.SIGTERM)
else:
os.remove(self.pid_file)
else:
print(f"[{self.name}] 尚未启动.", file=sys.stderr)
raise SystemExit(1)
def cli_run(self):
"""
命令行启动。
"""
if len(sys.argv) != 2:
print(f"Please use like: python3 {sys.argv[0]} [start|stop].", file=sys.stderr)
raise SystemExit(1)
if sys.argv[1] == 'start':
self.start()
elif sys.argv[1] == 'stop':
self.stop()
else:
print(f"未知命令: {sys.argv[1]}", file=sys.stderr)
raise SystemExit(1)
+152
View File
@@ -0,0 +1,152 @@
"""
服务管理包。用这个包能根据服务的名称启动服务。
"""
import importlib
import os.path
from types import ModuleType
from typing import Callable, Awaitable
import pandas as pd
from paste.core import aio_pool, config
from paste.core.logging import logger_config_name
from paste.util import ufile, udict
service_flag = ['service_name', 'pid_file', 'start_service', 'start', 'stop']
"""
模块若要成为服务,必须同时具备以上属性。
"""
def get_services(full_log_path=True):
"""
取得所有服务列表及其运行状态,返回格式为 Pandas DataFrame。
:param full_log_path: 是否输出完整日志文件路径
:return: 服务运行状态数据框架
"""
_service_info = []
_service_module = 'service'
_service_path = os.path.join(os.path.curdir, _service_module)
for _root, _dirs, _files in os.walk(_service_path):
for _file in _files:
_file_name, _ = os.path.splitext(_file)
if not _file_name.endswith('_service'):
continue
_mod_name = '.'.join([_service_module, _file_name])
_service_info.append(read_service_info(_mod_name, full_log_path))
_service_df = pd.DataFrame(_service_info)
return _service_df
def is_service(service_module: ModuleType):
"""
检查模块是否是服务模块。
:param service_module:
:return:
"""
_is_service = True
for _attr in service_flag:
if hasattr(service_module, _attr):
continue
_is_service = False
return _is_service
def read_service_info(full_module_name: str, full_log_path=True):
"""
读取模块信息。
:param full_module_name: 完整模块路径
:param full_log_path: 是否输出完整日志文件路径
:return: 服务模块信息
"""
_module = importlib.import_module(full_module_name)
assert is_service(_module), f"未设置关键属性,请确认是否为服务."
_pid = ''
_process_info = {}
_is_running = False
if os.path.exists(_module.pid_file):
_pid = ufile.read_to_buffer(_module.pid_file).decode('utf8').strip()
_process_info = aio_pool.process_info(int(_pid))
_is_running = True if _process_info else False
_configure = config.load_config()
_logger_config_name = getattr(_module, 'logger_config_name', logger_config_name)
if full_log_path:
_logger_file_path = os.path.abspath(udict.get_by_path(_configure, f"{_logger_config_name}.filename"))
else:
_logger_file_path = udict.get_by_path(_configure, f"{_logger_config_name}.filename")
# 代码中的 service_name 在这里作为服务描述
# 而 full_module_name 作为服务名称
_info = {
'service': full_module_name,
'service_name': _module.service_name,
'is_running': os.path.exists(_module.pid_file),
'logger_config_name': _logger_config_name,
'logger_file_path': _logger_file_path,
'pid_file': _module.pid_file,
'pid': _pid,
'process_name': _process_info.get('name', '') if _process_info else '',
'cpu_usage': _process_info.get('cpu_usage', '') if _process_info else '',
'memory_usage': _process_info.get('memory_usage', '') if _process_info else '',
'running_time': _process_info.get('running_time', '') if _process_info else '',
}
return _info
def start_service(full_module_name: str):
"""
在控制台启动服务,注意:当控制台关闭时,服务随即停止。
:param full_module_name: 完整模块路径。
:return: 操作状态,仅代表操作状态,并非立即启动服务
"""
_module = importlib.import_module(full_module_name)
_start: Callable = getattr(_module, 'start_service', None)
if not isinstance(_start, Callable):
return
_result = _start()
# 处理异步方法执行
if isinstance(_result, Awaitable):
_runner = aio_pool.get_aio_runner()
_result = _runner(_result)
def start(full_module_name: str):
"""
启动服务。
:param full_module_name: 完整模块路径。
:return: 操作状态,仅代表操作状态,并非立即启动服务
"""
_module = importlib.import_module(full_module_name)
_start: Callable = getattr(_module, 'start', None)
if not isinstance(_start, Callable):
return
_result = _start()
# 处理异步方法执行
if isinstance(_result, Awaitable):
_runner = aio_pool.get_aio_runner()
_result = _runner(_result)
def stop(full_module_name: str):
"""
停止服务。
:param full_module_name: 完整模块路径。
:return: 操作状态,仅代表操作状态,并非立即结束服务
"""
_module = importlib.import_module(full_module_name)
_stop: Callable = getattr(_module, 'stop', None)
if not isinstance(_stop, Callable):
return
_result = _stop()
# 处理异步方法执行
if isinstance(_result, Awaitable):
_runner = aio_pool.get_aio_runner()
_result = _runner(_result)
+496
View File
@@ -0,0 +1,496 @@
"""
系统服务,用于读取服务配置文件,启动或停止相关的服务。
"""
import asyncio
import datetime
import logging
from asyncio import AbstractEventLoop, Task
from enum import Enum
from typing import Callable, Awaitable, Optional
from dateutil.relativedelta import relativedelta
from paste.core import aio_pool
from paste.core.logging import echo_log
from paste.db.baseadapter import BaseAdapter
from paste.service.daemonize import DaemonizeService
class PeriodType(Enum):
WEEKLY = "weekly"
MONTHLY = "monthly"
YEARLY = "yearly"
QUARTERLY = "quarterly"
class TaskService:
"""
任务服务,专用于创建或停止服务。
"""
task_event_loop: Optional[AbstractEventLoop] = None
"""
任务件循环对象。
"""
def __init__(self, service_name: str = None, pid_file: str = None):
"""
构造函数。
:param service_name: 服务名称
:param pid_file: 进程 ID 文件路径
"""
self.service_name = service_name
"""
服务名称。
"""
if self.service_name is None:
self.service_name = '未命名服务'
self.pid_file = pid_file
"""
PID 文件路径。
"""
if self.pid_file is None:
_now = datetime.datetime.now()
self.pid_file = f'/tmp/task_service_{_now.strftime("%Y%m%d%H%M%S%f")}.pid'
self.task_list: list[Task] = []
"""
任务列表。
"""
self._create_task_params: list[dict] = []
"""
创建任务的参数列表。
"""
self.is_running = True
"""
是否允许运行。
"""
self.log_next_time = True
"""
是否记录下次执行时间。
"""
def event_loop(self):
"""
在需要调用的时间点取得事件循环对象。
:return: 事件循环对象
"""
self.task_event_loop = aio_pool.get_aio_loop()
return self.task_event_loop
def create_delay_task(self, fn: Callable = None, delay: int = 60, **kwargs):
"""
创建延时任务工厂,每次任务完成后,将等待固定时长后继续执行。
:param fn: 要执行的任务函数
:param delay: 延时长度,单位:秒
:param kwargs fn 函数的参数
"""
def log_next(log_next_time: bool):
if log_next_time and self.log_next_time:
echo_log(f"距下次执行:{fn.__name__} 还有:{delay} 秒.")
async def task_warp():
"""
任务包装器。
"""
if fn is not None:
try:
_result = fn(**kwargs)
if isinstance(_result, Awaitable):
await _result
except Exception as e:
echo_log(msg=e, level=logging.ERROR, is_log_exc=True)
async def task_loop():
"""
任务循环。
"""
if fn is None:
return
_log_next_time = True
_next_time = None
while self.is_running:
_delta_seconds = relativedelta(datetime.datetime.now(), _next_time).seconds if _next_time else 1
if _delta_seconds > 0:
await task_warp()
# 执行服务后,更新日期值
_next_time = datetime.datetime.now() + relativedelta(seconds=delay)
_log_next_time = True
else:
log_next(_log_next_time)
_log_next_time = False
await asyncio.sleep(0.5)
continue
_loop = self.event_loop()
_tsk: Task = _loop.create_task(task_loop())
return _tsk
def create_daily_task(self, fn: Callable = None, run_on_start=False,
year: int = None, month: int = None, day: int = None,
hour: int = None, minute: int = None, **kwargs):
"""
日常任务工厂,每次任务完成后,在第二天的固定时间继续执行。若设置的时间小于当前时间,则自动加一天。
:param fn: 要执行的任务函数
:param run_on_start: 是否在启动时立即运行一次,默认不运行
:param year: 年
:param month: 月
:param day: 日
:param hour: 时
:param minute: 分
:param kwargs fn 函数的参数
"""
_now = datetime.datetime.now()
year = _now.year if year is None else year
month = _now.month if month is None else month
day = _now.day if day is None else day
hour = _now.hour if hour is None else hour
minute = _now.minute if minute is None else minute
_next_time = datetime.datetime(year, month, day, hour, minute, 0)
if relativedelta(_next_time, datetime.datetime.now()).seconds < 0:
# 小于当前时间的,自动加一天
_next_time = _next_time + relativedelta(days=1)
def log_next(log_next_time: bool, next_time: datetime.datetime):
if log_next_time and self.log_next_time:
_delta = relativedelta(next_time, datetime.datetime.now())
_d, _h, _m, _s = _delta.days, _delta.hours, _delta.minutes, _delta.seconds
echo_log(f"距下次执行:{fn.__name__} 还有:{_d}{_h}{_m}{_s} 秒.")
async def task_warp():
"""
任务包装器。
"""
if fn is not None:
try:
_result = fn(**kwargs)
if isinstance(_result, Awaitable):
await _result
except Exception as e:
echo_log(msg=e, level=logging.ERROR, is_log_exc=True)
async def task_loop(next_time: datetime.datetime):
"""
任务循环。
:param next_time: 下次执行时间
"""
if fn is None:
return
_log_next_time = True
_run_on_start = run_on_start
while self.is_running:
_delta_seconds = relativedelta(datetime.datetime.now(), next_time).seconds
if _run_on_start or _delta_seconds > 0:
await task_warp()
_run_on_start = False
# 执行服务后,更新日期值
next_time = next_time + relativedelta(days=1)
_log_next_time = True
else:
log_next(_log_next_time, next_time)
_log_next_time = False
await asyncio.sleep(0.5)
continue
_loop = self.event_loop()
_tsk: Task = _loop.create_task(task_loop(next_time=_next_time))
return _tsk
def create_weekly_task(self, fn: Callable = None, weekday: int = 0,
hour: int = 0, minute: int = 0, run_on_start: bool = False, **kwargs):
"""每周某天固定时间执行(周一=0,周日=6)"""
return self.create_periodic_task(
fn, PeriodType.WEEKLY, run_on_start=run_on_start,
hour=hour, minute=minute, weekday=weekday, **kwargs
)
def create_monthly_task(self, fn: Callable = None, day_of_month: int = 1,
hour: int = 0, minute: int = 0, run_on_start: bool = False, **kwargs):
"""每月固定日期执行"""
return self.create_periodic_task(
fn, PeriodType.MONTHLY, run_on_start=run_on_start,
hour=hour, minute=minute, day_of_month=day_of_month, **kwargs
)
def create_yearly_task(self, fn: Callable = None, month: int = 1, day_of_month: int = 1,
hour: int = 0, minute: int = 0, run_on_start: bool = False, **kwargs):
"""每年固定日期执行"""
return self.create_periodic_task(
fn, PeriodType.YEARLY, run_on_start=run_on_start,
hour=hour, minute=minute, month=month, day_of_month=day_of_month, **kwargs
)
def create_quarterly_task(self, fn: Callable = None, start_month: int = 1, day_of_month: int = 1,
hour: int = 0, minute: int = 0, run_on_start: bool = False, **kwargs):
"""每季度固定日期执行(start_month: 1,4,7,10"""
return self.create_periodic_task(
fn, PeriodType.QUARTERLY, run_on_start=run_on_start,
hour=hour, minute=minute, month=start_month, day_of_month=day_of_month, **kwargs
)
def create_periodic_task(self, fn: Callable = None, period_type: PeriodType = PeriodType.WEEKLY,
run_on_start: bool = False, hour: int = 0, minute: int = 0,
weekday: Optional[int] = None, # 0=周一, 6=周日,仅 weekly 有效
day_of_month: Optional[int] = None, # 1-31,仅 monthly/quarterly/yearly 有效
month: Optional[int] = None, # 1-12,仅 quarterly/yearly 有效
**kwargs
):
"""
通用周期任务工厂。
:param fn: 任务函数
:param period_type: 周期类型 (weekly/monthly/yearly/quarterly)
:param run_on_start: 启动时是否立即运行一次
:param hour: 时 (0-23)
:param minute: 分 (0-59)
:param weekday: 星期几 (0=周一, 6=周日),仅 period_type=weekly 时使用
:param day_of_month: 每月几号 (1-31),仅 monthly/quarterly/yearly 时使用
:param month: 月份 (1-12),仅 quarterly/yearly 时使用(quarterly 时表示起始季度月份)
"""
def get_next_run_time(now: datetime.datetime) -> Optional[datetime.datetime]:
"""根据规则计算下一次执行时间"""
if period_type == PeriodType.WEEKLY:
if weekday is None:
raise ValueError("weekly 模式需要指定 weekday")
# 计算目标星期
days_ahead = (weekday - now.weekday()) % 7
# 如果今天就是目标星期
if days_ahead == 0:
target_time = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
# 如果今天的目标时间已过,则推迟到下周
if target_time <= now and not run_on_start:
days_ahead = 7
else:
return target_time
next_date = now + relativedelta(days=days_ahead)
return datetime.datetime(
next_date.year, next_date.month, next_date.day,
hour, minute, 0
)
elif period_type == PeriodType.MONTHLY:
if day_of_month is None:
raise ValueError("monthly 模式需要指定 day_of_month")
# 获取当前月份的最后一天
last_day = (now.replace(day=1) + relativedelta(months=1) - relativedelta(days=1)).day
target_day = min(day_of_month, last_day)
candidate = now.replace(day=target_day, hour=hour, minute=minute, second=0, microsecond=0)
# 如果候选时间已过,则下个月
if candidate <= now and not run_on_start:
next_month = now + relativedelta(months=1)
last_day_next = (next_month.replace(day=1) + relativedelta(months=1) - relativedelta(days=1)).day
target_day_next = min(day_of_month, last_day_next)
candidate = next_month.replace(day=target_day_next, hour=hour, minute=minute, second=0,
microsecond=0)
return candidate
elif period_type == PeriodType.QUARTERLY:
if day_of_month is None or month is None:
raise ValueError("quarterly 模式需要指定 day_of_month 和 month(起始季度月份)")
# 修正:计算季度的月份
q_months = []
for i in range(4):
qm = month + i * 3
if qm > 12:
qm -= 12
q_months.append(qm)
# 查找下一个季度月
target_month = None
target_year = now.year
for qm in sorted(q_months):
if qm > now.month:
target_month = qm
break
if target_month is None:
target_month = q_months[0]
target_year += 1
# 处理日期有效性
last_day = (datetime.datetime(target_year, target_month, 1) +
relativedelta(months=1) - relativedelta(days=1)).day
target_day = min(day_of_month, last_day)
candidate = datetime.datetime(target_year, target_month, target_day, hour, minute, 0)
if candidate <= now and not run_on_start:
# 跳到下个季度
candidate = candidate + relativedelta(months=3)
return candidate
elif period_type == PeriodType.YEARLY:
if day_of_month is None or month is None:
raise ValueError("yearly 模式需要指定 day_of_month 和 month")
# 检查年份
try:
candidate = datetime.datetime(now.year, month, day_of_month, hour, minute, 0)
except ValueError:
# 日期无效(如2月30日),取当月最后一天
last_day = (datetime.datetime(now.year, month, 1) +
relativedelta(months=1) - relativedelta(days=1)).day
candidate = datetime.datetime(now.year, month, last_day, hour, minute, 0)
if candidate <= now and not run_on_start:
try:
candidate = datetime.datetime(now.year + 1, month, day_of_month, hour, minute, 0)
except ValueError:
last_day = (datetime.datetime(now.year + 1, month, 1) +
relativedelta(months=1) - relativedelta(days=1)).day
candidate = datetime.datetime(now.year + 1, month, last_day, hour, minute, 0)
return candidate
async def task_warp():
if fn is not None:
try:
_result = fn(**kwargs)
if isinstance(_result, Awaitable):
await _result
except Exception as e:
echo_log(msg=e, level=logging.ERROR, is_log_exc=True)
async def task_loop():
if fn is None:
return
_log_next_time = True
_run_on_start = run_on_start
next_time = get_next_run_time(datetime.datetime.now())
while self.is_running:
now = datetime.datetime.now()
if _run_on_start or now >= next_time:
await task_warp()
_run_on_start = False
# 执行完后,基于当前时间重新计算下一次
next_time = get_next_run_time(now)
_log_next_time = True
else:
if _log_next_time and self.log_next_time:
delta = relativedelta(next_time, now)
echo_log(
f"距下次执行:{fn.__name__} 还有:{delta.days}{delta.hours}{delta.minutes}{delta.seconds}")
_log_next_time = False
await asyncio.sleep(1)
_loop = self.event_loop()
return _loop.create_task(task_loop())
def add_task(self, creator: Callable, fn: Callable, **kwargs):
"""
添加任务。注意:这里只是存储创建参数,直到任务启动前,才实际把任务创建出来。
:param creator: 任务工厂,对应:应延时任务工厂、日常任务工厂
:param fn: 任务函数,即任务对应的实际功能函数
:param kwargs: 任务函数的参数
"""
_d = {
'creator': creator, # 创建器,对应延时任务和日常任务
'fn': fn, 'kwargs': kwargs # 任务函数及其参数
}
self._create_task_params.append(_d)
def rebuild_task_list(self):
"""
重建任务列表。
:return: 任务列表
"""
self.task_list.clear()
# 遍历创建器列表,创建任务
for _ctp in self._create_task_params:
_creator: Callable = _ctp.get('creator')
_fn: Callable = _ctp.get('fn')
_kwargs: dict = _ctp.get('kwargs')
_task = _creator(_fn, **_kwargs)
self.task_list.append(_task)
return self.task_list
async def run_tasks(self):
"""
执行任务。支持多任务同时启动,如:预统计服务、设备数据同步服务等。
"""
try:
self.rebuild_task_list()
echo_log(f'{self.service_name}启动成功.')
await asyncio.gather(*self.task_list)
except Exception as e:
echo_log(msg=e, level=logging.ERROR, is_log_exc=True)
def start_service(self, env_check: bool = True):
"""
以控制台服务方式启动服务。
"""
echo_log(f'正在启动{self.service_name}...')
try:
if env_check:
# 检测 MySQL 服务是否正常
echo_log('检测 Database 服务...')
BaseAdapter.ping()
# 注意,这里是取得协程
_future = self.run_tasks()
# 开始执行任务事件循环
_loop = self.event_loop()
_loop.run_until_complete(_future)
except KeyboardInterrupt:
echo_log(msg='KeyboardInterrupt')
self.stop_service()
except Exception as e:
echo_log(msg=e, level=logging.ERROR, is_log_exc=True)
def stop_service(self):
"""
停止服务。
"""
self.is_running = False
echo_log(f'{self.service_name}已停止.')
def start(self, env_check: bool = True):
"""
以驻内存服务方式启动服务。
"""
ds = DaemonizeService(pid_file=self.pid_file, name=f'{self.service_name}')
ds.set_start_callback(self.start_service, env_check=env_check)
ds.set_term_callback(self.stop_service)
ds.start()
def stop(self, env_check: bool = True):
"""
停止驻内存服务。
"""
ds = DaemonizeService(pid_file=self.pid_file, name=f'{self.service_name}')
ds.set_start_callback(self.start_service, env_check=env_check)
ds.set_term_callback(self.stop_service)
ds.stop()