Compare commits

...

2 Commits

Author SHA1 Message Date
cae41d9bb0 update VideoCompress 2026-01-11 12:40:07 +08:00
6f304a634c add README in calc utils 2026-01-11 11:43:35 +08:00
7 changed files with 577 additions and 5928 deletions

View File

@ -1,22 +1,22 @@
{ {
"save_to": "single", "save_to": "single",
"crf": 18,
"bitrate": null, "bitrate": null,
"crf": 26, "codec": "h264",
"codec": "h264_qsv", "hwaccel": null,
"hwaccel": "qsv",
"extra": [], "extra": [],
"ffmpeg": "C:/tools/ffmpeg/bin/ffmpeg.exe", "ffmpeg": "ffmpeg",
"ffprobe": "C:/tools/ffmpeg/bin/ffprobe", "ffprobe": "ffprobe",
"manual": null, "manual": null,
"video_ext": [ "video_ext": [
".mp4", ".mp4",
".mkv" ".mkv"
], ],
"compress_dir_name": "compress_qsv", "compress_dir_name": "compress",
"resolution": null, "resolution": null,
"fps": "30", "fps": 30,
"test_video_resolution": "1920x1080", "test_video_resolution": "1920x1080",
"test_video_fps": "30", "test_video_fps": 30,
"test_video_input": "compress_video_test.mp4", "test_video_input": "compress_video_test.mp4",
"test_video_output": "compressed_video_test.mp4", "test_video_output": "compressed_video_test.mp4",
"disable_hwaccel_when_fail": true "disable_hwaccel_when_fail": true

View File

@ -122,11 +122,15 @@ def get_video_frame_count(
} }
for key in fallback_order: for key in fallback_order:
try:
try: try:
func = methods.get(key) func = methods.get(key)
if not func: if not func:
continue continue
n = func(path, stream_index) n = func(path, stream_index)
except Exception:
logging.debug(f"Errored to get frame with {key}.",exc_info=True)
continue
if isinstance(n, int) and n >= 0: if isinstance(n, int) and n >= 0:
return n return n
else: else:

View File

@ -8,23 +8,27 @@ from time import time
from rich.logging import RichHandler from rich.logging import RichHandler
from rich.progress import Progress from rich.progress import Progress
from pickle import dumps, loads from pickle import dumps, loads
from typing import Optional, Callable,Literal from typing import Optional, Callable, Literal, List, Any, TYPE_CHECKING
import atexit import atexit
import re import re
import get_frame import get_frame
import json
try:
from pydantic import BaseModel, Field, field_validator, model_validator from pydantic import BaseModel, Field, field_validator, model_validator
HAS_PYDANTIC = True
class Config(BaseModel): class Config(BaseModel):
save_to: Literal["single", "multi"] = Field("single", description="保存到单文件夹或者每个子文件夹创建compress_dir") save_to: Literal["single", "multi"] = Field("single", description="保存到单文件夹或者每个子文件夹创建compress_dir")
crf: Optional[int] = Field(None, ge=0, le=51, description="CRF值范围0-51") crf: Optional[int] = Field(18, ge=0, le=51, description="CRF值范围0-51")
bitrate: Optional[str] = Field(None, description="比特率,格式如: 1000k, 2.5M, 1500B") bitrate: Optional[str] = Field(None, description="比特率,格式如: 1000k, 2.5M, 1500B")
codec: str = Field("h264", description="ffmpeg的codec如果使用GPU需要对应设置") codec: str = Field("h264", description="ffmpeg的codec如果使用GPU需要对应设置")
hwaccel: Optional[Literal["amf", "qsv", "cuda"]] = Field(None, description="使用GPU加速") hwaccel: Optional[Literal["amf", "qsv", "cuda"]] = Field(None, description="使用GPU加速")
extra:Optional[list[str]] = Field(None,description="插入到ffmpeg输出前的自定义参数") extra: List[str] = Field([], description="插入到ffmpeg输出前的自定义参数")
ffmpeg: str = "ffmpeg" ffmpeg: str = "ffmpeg"
ffprobe: str = "ffprobe" ffprobe: str = "ffprobe"
manual:Optional[list[str]] = Field(None,description=r"手动设置ffmpeg命令ffmpeg -i {input} {manual} {output}") manual: Optional[List[str]] = Field(None, description=r"手动设置ffmpeg命令ffmpeg -i {input} {manual} {output}")
video_ext:list[str] = Field([".mp4", ".mkv"],description="视频文件后缀,含.") video_ext: List[str] = Field([".mp4", ".mkv"], description="视频文件后缀,含.")
compress_dir_name: str = Field("compress", description="压缩文件夹名称") compress_dir_name: str = Field("compress", description="压缩文件夹名称")
resolution: Optional[str] = Field(None, description="统一到特定尺寸None为不使用缩放") resolution: Optional[str] = Field(None, description="统一到特定尺寸None为不使用缩放")
fps: int = Field(30, description="fps", ge=0) fps: int = Field(30, description="fps", ge=0)
@ -34,8 +38,6 @@ class Config(BaseModel):
test_video_output: str = "compressed_video_test.mp4" test_video_output: str = "compressed_video_test.mp4"
disable_hwaccel_when_fail: bool = Field(True, description="当运行失败时,禁用硬件加速") disable_hwaccel_when_fail: bool = Field(True, description="当运行失败时,禁用硬件加速")
@field_validator('bitrate') @field_validator('bitrate')
@classmethod @classmethod
def validate_bitrate(cls, v: Optional[str]) -> Optional[str]: def validate_bitrate(cls, v: Optional[str]) -> Optional[str]:
@ -57,15 +59,12 @@ class Config(BaseModel):
return v return v
@field_validator("compress_dir_name") @field_validator("compress_dir_name")
# @field_validator("test_video_input")
# @field_validator("test_video_output")
@classmethod @classmethod
def valid_path(cls, v: str) -> str: def valid_path(cls, v: str) -> str:
if re.search(r'[\\/:*?"<>|\x00-\x1F]', v): if re.search(r'[\\/:*?"<>|\x00-\x1F]', v):
raise ValueError("某配置不符合目录名语法") raise ValueError("某配置不符合目录名语法")
return v return v
@model_validator(mode='after') @model_validator(mode='after')
def validate_mutual_exclusive(self): def validate_mutual_exclusive(self):
crf_none = self.crf is None crf_none = self.crf is None
@ -77,31 +76,98 @@ class Config(BaseModel):
return self return self
def dump(self):
return self.model_dump()
except ImportError:
HAS_PYDANTIC = False
from dataclasses import dataclass, asdict
import copy
@dataclass
class Config:
save_to: str = "single"
crf: Optional[int] = 18
bitrate: Optional[str] = None
codec: str = "h264"
hwaccel: Optional[str] = None
extra: List[str] = []
ffmpeg: str = "ffmpeg"
ffprobe: str = "ffprobe"
manual: Optional[List[str]] = None
video_ext: List[str] = [".mp4", ".mkv"]
compress_dir_name: str = "compress"
resolution: Optional[str] = None
fps: int = 30
test_video_resolution: str = "1920x1080"
test_video_fps: int = 30
test_video_input: str = "compress_video_test.mp4"
test_video_output: str = "compressed_video_test.mp4"
disable_hwaccel_when_fail: bool = True
def update(self, other):
if isinstance(other, dict):
d = other
elif isinstance(other, Config):
d = asdict(other)
else:
return
for k, v in d.items():
if hasattr(self, k):
setattr(self, k, v)
def copy(self):
return copy.deepcopy(self)
def dump(self):
return asdict(self)
root = None root = None
if os.environ.get("INSTALL", "0") == "1": if os.environ.get("INSTALL", "0") == "1":
CFG_FILE = Path(os.getenv("APPDATA", "C:/")) / "VideoCompress" / "config.json" CFG_FILE = Path(os.getenv("APPDATA", "C:/")) / "VideoCompress" / "config.json"
else: else:
CFG_FILE = Path(sys.path[0]) / "config.json" CFG_FILE = Path(sys.path[0]) / "config.json"
CFG = {
"save_to": "single", if CFG_FILE.exists():
"crf": "18", try:
"bitrate": None, import json
"codec": "h264",
"hwaccel": None, if HAS_PYDANTIC:
"extra": [], assert BaseModel # type: ignore
"ffmpeg": "ffmpeg", assert issubclass(Config, BaseModel)
"manual": None, CFG = Config.model_validate_json(CFG_FILE.read_text())
"video_ext": [".mp4", ".mkv"], else:
"compress_dir_name": "compress", assert Config
"resolution": None, cfg:dict[str, Any] = json.loads(CFG_FILE.read_text())
"fps": "30", CFG = Config(**cfg)
"test_video_resolution": "1920x1080",
"test_video_fps": "30", get_frame.ffprobe = CFG.ffprobe
"test_video_input": "compress_video_test.mp4", logging.debug(CFG)
"test_video_output": "compressed_video_test.mp4", except KeyboardInterrupt as e:
"disable_hwaccel_when_fail": True, raise e
} except Exception as e:
logging.warning("Invalid config file, ignored.")
logging.debug(e)
else:
try:
if HAS_PYDANTIC:
if TYPE_CHECKING:
assert BaseModel # type: ignore
assert issubclass(Config, BaseModel)
CFG = Config() # type: ignore
CFG_FILE.write_text(CFG.model_dump_json(indent=4))
else:
import json
if TYPE_CHECKING:
assert Config
assert asdict # type: ignore
CFG = Config() # type: ignore
CFG_FILE.write_text(json.dumps(asdict(CFG), indent=4))
logging.info("Config file created.")
except Exception as e:
logging.warning("Failed to create config file.", exc_info=e)
def get_cmd(video_path: str | Path, output_file: str | Path) -> list[str]: def get_cmd(video_path: str | Path, output_file: str | Path) -> list[str]:
@ -110,23 +176,23 @@ def get_cmd(video_path: str | Path, output_file: str | Path) -> list[str]:
if isinstance(output_file, Path): if isinstance(output_file, Path):
output_file = str(output_file.resolve()) output_file = str(output_file.resolve())
if CFG["manual"] is not None: if CFG.manual is not None:
command = [CFG["ffmpeg"], "-hide_banner", "-i", video_path] command = [CFG.ffmpeg, "-hide_banner", "-i", video_path]
command.extend(CFG["manual"]) command.extend(CFG.manual)
command.append(output_file) command.append(output_file)
return command return command
command = [ command = [
CFG["ffmpeg"], CFG.ffmpeg,
"-hide_banner", "-hide_banner",
] ]
if CFG["hwaccel"] is not None: if CFG.hwaccel is not None:
command.extend( command.extend(
[ [
"-hwaccel", "-hwaccel",
CFG["hwaccel"], CFG.hwaccel,
"-hwaccel_output_format", "-hwaccel_output_format",
CFG["hwaccel"], CFG.hwaccel,
] ]
) )
@ -137,47 +203,47 @@ def get_cmd(video_path: str | Path, output_file: str | Path) -> list[str]:
] ]
) )
if CFG["bitrate"] is not None: if CFG.bitrate is not None:
if CFG["resolution"] is not None: if CFG.resolution is not None:
command.extend( command.extend(
[ [
"-vf", "-vf",
f"scale={CFG['resolution']}", f"scale={CFG.resolution}",
] ]
) )
command.extend( command.extend(
[ [
"-c:v", "-c:v",
CFG["codec"], CFG.codec,
"-b:v", "-b:v",
CFG["bitrate"], CFG.bitrate,
"-r", "-r",
str(CFG["fps"]), str(CFG.fps),
"-y", "-y",
] ]
) )
else: else:
if CFG["resolution"] is not None: if CFG.resolution is not None:
command.extend( command.extend(
[ [
"-vf", "-vf",
f"scale={CFG['resolution']}", f"scale={CFG.resolution}",
] ]
) )
command.extend( command.extend(
[ [
"-c:v", "-c:v",
CFG["codec"], CFG.codec,
"-global_quality", "-global_quality",
str(CFG["crf"]), str(CFG.crf),
"-r", "-r",
str(CFG["fps"]), str(CFG.fps),
"-y", "-y",
] ]
) )
command.extend(CFG["extra"]) command.extend(CFG.extra)
command.append(output_file) command.append(output_file)
logging.debug(f"Create CMD: {command}") logging.debug(f"Create CMD: {command}")
return command return command
@ -219,7 +285,7 @@ def process_video(
if compress_dir is None: if compress_dir is None:
# 在视频文件所在目录下创建 compress 子目录(如果不存在) # 在视频文件所在目录下创建 compress 子目录(如果不存在)
compress_dir = video_path.parent / CFG["compress_dir_name"] compress_dir = video_path.parent / CFG.compress_dir_name
else: else:
assert root assert root
compress_dir /= video_path.parent.relative_to(root) compress_dir /= video_path.parent.relative_to(root)
@ -277,7 +343,7 @@ def process_video(
assert result.stdout is not None assert result.stdout is not None
logging.debug(result.stdout.read()) logging.debug(result.stdout.read())
logging.debug(total) logging.debug(total)
if CFG["hwaccel"] == "mediacodec" and CFG["codec"] in [ if CFG.hwaccel == "mediacodec" and CFG.codec in [
"h264_mediacodec", "h264_mediacodec",
"hevc_mediacodec", "hevc_mediacodec",
]: ]:
@ -285,32 +351,36 @@ def process_video(
"mediacodec硬件加速器已知在较短片段上存在异常将禁用加速重试。" "mediacodec硬件加速器已知在较短片段上存在异常将禁用加速重试。"
) )
output_file.unlink(missing_ok=True) output_file.unlink(missing_ok=True)
bak = CFG.copy() bak = CFG.codec, CFG.hwaccel
CFG["hwaccel"] = None CFG.hwaccel = None
CFG["codec"] = "h264" if CFG["codec"] == "h264_mediacodec" else "hevc" CFG.codec = "h264" if CFG.codec == "h264_mediacodec" else "hevc"
assert not output_file.exists() assert not output_file.exists()
ret = process_video(video_path, compress_dir, update_func) ret = process_video(video_path, compress_dir, update_func)
CFG.update(bak) CFG.codec, CFG.hwaccel = bak
if not ret: if not ret:
logging.error("重试仍然失败。") logging.error("重试仍然失败。")
return False return False
else: else:
return True return True
elif CFG["disable_hwaccel_when_fail"] and CFG["hwaccel"] is not None: elif CFG.disable_hwaccel_when_fail and CFG.hwaccel is not None:
logging.info("正在禁用硬件加速器重试,进度条可能发生混乱。") logging.info("正在禁用硬件加速器重试,进度条可能发生混乱。")
output_file.unlink(missing_ok=True) output_file.unlink(missing_ok=True)
bak = CFG.copy() if TYPE_CHECKING:
CFG["hwaccel"] = None assert BaseModel # type: ignore
assert isinstance(CFG, BaseModel)
bak = CFG.codec, CFG.hwaccel
CFG.hwaccel = None
if ( if (
CFG["codec"].endswith("_mediacodec") CFG.codec.endswith("_mediacodec")
or CFG["codec"].endswith("_qsv") or CFG.codec.endswith("_qsv")
or CFG["codec"].endswith("_nvenc") or CFG.codec.endswith("_nvenc")
or CFG["codec"].endswith("_amf") or CFG.codec.endswith("_amf")
): ):
CFG["codec"] = CFG["codec"].split("_")[0] CFG.codec = CFG.codec.split("_")[0]
assert not output_file.exists() assert not output_file.exists()
ret = process_video(video_path, compress_dir, update_func) ret = process_video(video_path, compress_dir, update_func)
CFG.update(bak) CFG.codec, CFG.hwaccel = bak
if not ret: if not ret:
logging.error("重试仍然失败。") logging.error("重试仍然失败。")
return False return False
@ -329,7 +399,7 @@ def process_video(
def traverse_directory(root_dir: Path): def traverse_directory(root_dir: Path):
video_extensions = set(CFG["video_ext"]) video_extensions = set(CFG.video_ext)
sm = None sm = None
# 获取视频文件列表和帧数信息 # 获取视频文件列表和帧数信息
video_files:list[Path] = [] video_files:list[Path] = []
@ -338,8 +408,8 @@ def traverse_directory(root_dir: Path):
d = que.pop() d = que.pop()
for file in d.glob("*") if d.is_dir() else [d]: for file in d.glob("*") if d.is_dir() else [d]:
if ( if (
file.parent.name == CFG["compress_dir_name"] file.parent.name == CFG.compress_dir_name
or file.name == CFG["compress_dir_name"] or file.name == CFG.compress_dir_name
): ):
continue continue
if file.is_file() and file.suffix.lower() in video_extensions: if file.is_file() and file.suffix.lower() in video_extensions:
@ -426,9 +496,9 @@ def traverse_directory(root_dir: Path):
) )
prog.update(main_task, completed=completed_start + x) prog.update(main_task, completed=completed_start + x)
if CFG["save_to"] == "single": if CFG.save_to == "single":
process_video( process_video(
file, root_dir / CFG["compress_dir_name"], update_progress file, root_dir / CFG.compress_dir_name, update_progress
) )
else: else:
process_video(file, None, update_progress) process_video(file, None, update_progress)
@ -450,7 +520,7 @@ def test():
try: try:
subprocess.run( subprocess.run(
[CFG["ffmpeg"], "-version"], stdout=-3, stderr=-3 [CFG.ffmpeg, "-version"], stdout=-3, stderr=-3
).check_returncode() ).check_returncode()
except Exception as e: except Exception as e:
print(__file__) print(__file__)
@ -458,7 +528,7 @@ def test():
exit(-1) exit(-1)
try: try:
ret = subprocess.run( ret = subprocess.run(
f"{CFG['ffmpeg']} -hide_banner -f lavfi -i testsrc=duration=1:size={CFG['test_video_resolution']}:rate={CFG['test_video_fps']} -c:v libx264 -y -pix_fmt yuv420p {CFG['test_video_input']}".split(), f"{CFG.ffmpeg} -hide_banner -f lavfi -i testsrc=duration=1:size={CFG.test_video_resolution}:rate={CFG.test_video_fps} -c:v libx264 -y -pix_fmt yuv420p {CFG.test_video_input}".split(),
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, text=True,
@ -469,8 +539,8 @@ def test():
logging.debug(ret.stderr) logging.debug(ret.stderr)
ret.check_returncode() ret.check_returncode()
cmd = get_cmd( cmd = get_cmd(
CFG["test_video_input"], CFG.test_video_input,
CFG["test_video_output"], CFG.test_video_output,
) )
ret = subprocess.run( ret = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
@ -512,30 +582,6 @@ def main(_root=None):
logging.info("-------------------------------") logging.info("-------------------------------")
logging.info(datetime.now().strftime("Video Compress started at %Y/%m/%d %H:%M")) logging.info(datetime.now().strftime("Video Compress started at %Y/%m/%d %H:%M"))
if CFG_FILE.exists():
try:
import json
cfg: dict = json.loads(CFG_FILE.read_text())
cfg_model = Config(**cfg)
CFG.update(cfg_model.model_dump())
get_frame.ffprobe = CFG["ffprobe"]
logging.debug(cfg_model)
logging.debug(CFG)
except KeyboardInterrupt as e:
raise e
except Exception as e:
logging.warning("Invalid config file, ignored.")
logging.debug(e)
else:
try:
import json
CFG_FILE.write_text(json.dumps(CFG, indent=4))
logging.info("Config file created.")
except Exception as e:
logging.warning("Failed to create config file.", exc_info=e)
if _root is not None: if _root is not None:
root = Path(_root) root = Path(_root)
else: else:
@ -546,7 +592,7 @@ def main(_root=None):
sys.exit(1) sys.exit(1)
root = Path(sys.argv[1]) root = Path(sys.argv[1])
if root.name.lower() == CFG["compress_dir_name"].lower(): if root.name.lower() == CFG.compress_dir_name.lower():
logging.critical("请修改目标目录名为非compress。") logging.critical("请修改目标目录名为非compress。")
logging.error("Error termination via invalid input.") logging.error("Error termination via invalid input.")
sys.exit(1) sys.exit(1)
@ -576,5 +622,5 @@ def main(_root=None):
if __name__ == "__main__": if __name__ == "__main__":
sys.argv.append(r'C:\Users\flt\Documents\WeChat Files\wxid_m8h0igh8p52p22\FileStorage\Video') # sys.argv.append(r'C:\Users\flt\Documents\WeChat Files\wxid_m8h0igh8p52p22\FileStorage\Video')
main() main()

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,73 @@
# calc-utils
个人使用的计算化学工具集,主要基于 [ASE (Atomic Simulation Environment)](https://wiki.fysik.dtu.dk/ase/) 和 [RDKit](https://www.rdkit.org/)。
包含了一些方便的转换工具以及针对特定服务器环境PBS/Slurm/Custom定制的 `ase.calculators.gaussian` 补丁。
专用软件,`futils.gaussian`在不同服务器环境中无法直接运行,必须予以修改。
## 安装
需要 Python 3.12+。
```bash
git clone https://github.com/your-repo/calc-utils.git
cd calc-utils
pip install .
```
## 功能模块
### 1. `futils.gaussian` (**Breaking Change**)
这是一个对 `ase.calculators.gaussian` 的深度定制和 Monkey Patch。
**注意:导入此模块会直接修改 `ase.calculators.gaussian` 中的类定义。**
主要修改内容:
- **强制任务提交脚本**:计算器的 `command` 默认被设置为调用 `gsub_wait` 脚本。
- 默认路径硬编码为 `/home/fanhj/calcs/lele/tools/gsub_wait`(需要在 `futils/gaussian.py` 中按需修改 `GSUB` 变量)。
- **文件后缀变更**:输入文件使用 `.gin` 而非 `.gjf`,输出文件默认读取 `.out`
- **参数增强**`__init__` 方法提供了更详细的 Type Hinting 和默认参数(如 `mem="30GB"`, `proc=32`)。
- **辅助方法**:增加了 `mod()` 方法用于快速复制并修改计算器参数。
```python
from futils.gaussian import Gaussian
from ase import Atoms
# 使用定制后的 Gaussian 计算器
# 注意:这会尝试调用 gsub_wait 提交任务
calc = Gaussian(label='test_calc', method='B3LYP', basis='6-31G(d)')
```
### 2. `futils.rdkit2ase`
提供 RDKit 分子对象 (`rdkit.Chem.Mol`) 与 ASE 原子对象 (`ase.Atoms`) 之间的无缝转换,**保留 3D 坐标**。
```python
from futils.rdkit2ase import MolToAtoms, AtomsToMol
from rdkit import Chem
# RDKit -> ASE
mol = Chem.MolFromMolFile("molecule.mol")
atoms = MolToAtoms(mol)
# ASE -> RDKit
new_mol = AtomsToMol(atoms)
```
### 3. `futils.rdkit_utils`
一些 RDKit 绘图辅助函数。
- `draw2D(mol)`: 生成 SVG 格式的 2D 分子图。
- `draw3D(mol)`: 使用 IPythonConsole 绘制 3D 分子图。
## 脚本工具 (`bin/`)
本项目包含了一些用于任务提交管理的 Shell 脚本,适用于特定的集群环境。
- **`gsub`**: 任务提交脚本。支持本地或通过 SSH 远程提交到名为 `cluster` 的主机。
- **`gsub_wait`**: 提交任务并阻塞等待完成,用于 ASE Calculator 的 `command` 调用,以便实现 Python 脚本的同步执行。
**配置说明**
使用前请检查 `bin/` 下的脚本以及 `futils/gaussian.py` 中的 `GSUB` 路径,根据您的服务器环境进行调整。

176
calc_utils/bin/gsub Normal file
View File

@ -0,0 +1,176 @@
#!/bin/bash
set -u
# Usage: gsub <jobname>
job=${1:-}
if [[ -z "$job" ]]; then
echo "Usage: $0 <jobname-without-extension>"
exit 1
fi
# ==========================================
# 0. 安全检测函数 (Safety Check)
# ==========================================
check_dangerous_path() {
local path="${1:-}"
# 1. Empty check
if [[ -z "$path" ]]; then
echo "Error: Empty path is dangerous for deletion." >&2
return 1
fi
# 2. Root check
if [[ "$path" == "/" ]]; then
echo "Error: Root path '/' is dangerous for deletion." >&2
return 1
fi
# 3. Space check (optional, but good for safety)
if [[ "$path" =~ ^[[:space:]]+$ ]]; then
echo "Error: Whitespace path is dangerous." >&2
return 1
fi
return 0
}
# ==========================================
# 1. 检查运行环境 (Check Host)
# ==========================================
# 如果不是 cluster尝试通过 SSH 远程调用
host_short=$(hostname -s 2>/dev/null || hostname)
if [[ "$host_short" != "cluster" ]]; then
# 假设本地挂载路径 /mnt/home 对应远程 /home (根据原脚本逻辑调整)
cur_dir=$(pwd)
remote_dir="${cur_dir//\/mnt\/home/\/home}"
# 定位当前脚本并转换为远程路径
# 获取脚本所在目录的绝对路径
script_dir=$(cd "$(dirname "$0")" && pwd)
script_name=$(basename "$0")
local_script="$script_dir/$script_name"
# 同样对脚本路径进行替换
remote_script="${local_script//\/mnt\/home/\/home}"
# 尝试在远程执行自己
echo "Running remotely on cluster: $remote_script" >&2
ssh cluster "cd '$remote_dir' && '$remote_script' '$job'"
exit $?
fi
# ==========================================
# 2. 准备作业 (Prepare Job)
# ==========================================
gin_file="$job.gin"
if [[ ! -f "$gin_file" ]]; then
echo "Error: $gin_file not found in $(pwd)"
exit 2
fi
# 解析配置确定资源 (Parse Proc)
# 查找 %NProcShared=XX
proc=$(sed -n 's/^%NProcShared=\([0-9]\+\).*$/\1/pI' "$gin_file" | head -n 1)
queue=""
ppn=""
if [[ "$proc" == "32" ]]; then
queue="n32"
ppn="32"
elif [[ "$proc" == "20" ]]; then
queue="n20"
ppn="20"
else
echo "Error: Unsupported NProcShared=$proc in $gin_file. Only 20 or 32 allowed."
exit 1
fi
# 清理旧文件 (Clean up old output)
if [[ -f "$job.out" ]]; then
# 原脚本逻辑:休眠并删除
# echo "Warning: $job.out exists. Deleting..." >&2
# 使用安全检查
if check_dangerous_path "$job.out"; then
rm "$job.out"
else
echo "Skipping deletion of unsafe path: $job.out" >&2
exit 1
fi
fi
# ==========================================
# 3. 生成作业脚本 (.job)
# ==========================================
job_file="$job.job"
# 使用 heredoc 动态生成 PBS 脚本
# 整合了原 g16_32.pbs 的内容和 gsub32 的追加内容
cat > "$job_file" <<EOF
#!/bin/sh
#PBS -l nodes=1:ppn=$ppn
#PBS -q $queue
#PBS -j oe
#PBS -N $job
cd \$PBS_O_WORKDIR
# Define Safety Check Function in Job Script
check_rm_path() {
p="\$1"
# Empty check
if [ -z "\$p" ]; then
echo "Refusing to delete empty path"
return 1
fi
# Root check
if [ "\$p" = "/" ]; then
echo "Refusing to delete root path"
return 1
fi
return 0
}
export g16root=/share/apps/soft
source \$g16root/g16/bsd/g16.profile
# Create Scratch Directory
if [ -n "\$USER" ] && [ -n "\$PBS_JOBID" ]; then
mkdir -p /scr/\$USER/\$PBS_JOBID
export GAUSS_SCRDIR=/scr/\$USER/\$PBS_JOBID
else
echo "Error: USER or PBS_JOBID not set. Cannot setup scratch."
exit 1
fi
NODES=\`cat \$PBS_NODEFILE | uniq\`
echo "--------------------------------------------------------"
echo " JOBID: \$PBS_JOBID"
echo " The job was started at \`date\`"
echo " The job was running at \$NODES."
echo "--------------------------------------------------------"
# Run G16 Job
echo "Executing: g16 < $gin_file > $job.out"
g16 < $gin_file > $job.out
echo "--------------------------------------------------------"
echo " The job was finished at \`date\`"
echo "--------------------------------------------------------"
# Delete the tmp File (Cleanup Scratch)
echo "Cleaning up \$GAUSS_SCRDIR"
if check_rm_path "\$GAUSS_SCRDIR"; then
rm -rf "\$GAUSS_SCRDIR"
fi
EOF
# ==========================================
# 4. 提交作业 (Submit)
# ==========================================
# qsub 会输出 Job ID例如 12345.cluster
qsub "$job_file"

116
calc_utils/bin/gsub_wait Normal file
View File

@ -0,0 +1,116 @@
#!/bin/bash
set -u
# Usage: gsub_wait <jobname>
job=${1:-}
if [[ -z "$job" ]]; then
echo "Usage: $0 <jobname-without-extension>"
exit 1
fi
# ==========================================
# 1. 提交任务 (Submit Job)
# ==========================================
# 确定 gsub 命令位置
# 优先查找当前目录下的 gsub否则查找 PATH
if [[ -x "./gsub" ]]; then
GSUB_CMD="./gsub"
else
GSUB_CMD="gsub"
fi
# 调用 gsub 并捕获输出
# 注意gsub 内部可能通过 SSH 在远程执行,最终返回 qsub 的输出
output=$($GSUB_CMD "$job")
echo "$output"
# ==========================================
# 2. 检查是否需要等待 (Check Silent Mode)
# ==========================================
# 如果 GSUB_SILENT 为 1则不进行监控直接退出
if [[ "${GSUB_SILENT:-0}" == "1" ]]; then
exit 0
fi
# ==========================================
# 3. 监控任务进度 (Monitor Progress)
# ==========================================
# 尝试提取 Job ID (例如: 67147.cluster -> 67147)
jobid_full=$(echo "$output" | grep -oE '[0-9]+\.cluster|[0-9]+' | head -n 1 || true)
if [[ -n "$jobid_full" ]]; then
jobid=${jobid_full%%.*}
# 准备参数
out_file="$job.out"
gin_file="$job.gin"
end_file="$job.job.o$jobid"
if [[ ! -f "$gin_file" ]]; then
# 如果 gin 文件找不到(可能是远程路径问题?),跳过监控
echo "Warning: $gin_file not found nearby. Skipping monitor."
exit 0
fi
# 计算 Total Steps: (--link1-- 数量) + 1
link_count=$(grep -c -- "--link1--" "$gin_file" || true)
total=$((link_count + 1))
cntDone=0
cntSCF=0
last_lines=0
echo "Monitoring Job $jobid..."
while true; do
# A. 检查 PBS 结束文件 (Job 完成标志)
if [[ -f "$end_file" ]]; then
echo "Job finished (found $end_file)."
break
fi
# B. 检查并读取 .out 输出文件
if [[ -f "$out_file" ]]; then
curr_lines=$(wc -l < "$out_file" 2>/dev/null || echo 0)
# 如果文件变小(被截断或重新生成),重置读取位置
if (( curr_lines < last_lines )); then last_lines=0; fi
if (( curr_lines > last_lines )); then
# 逐行处理新增内容
# 使用进程替换 < <(...) 避免管道导致的子shell变量丢失问题
while IFS= read -r line; do
# 检查 SCF Done
# 正则匹配: SCF Done: ... E ... = (数值) A.U.
if [[ "$line" =~ SCF[[:space:]]Done:.*E.*=[[:space:]]*([-0-9.]+)[[:space:]]*A\.U\. ]]; then
energy="${BASH_REMATCH[1]}"
cntSCF=$((cntSCF + 1))
echo "$job: SCF Done: $energy [$cntSCF] ($cntDone/$total)"
fi
# 检查 Termination
if [[ "$line" == *"termination of Gaussian"* ]]; then
cntDone=$((cntDone + 1))
echo "$job: task done ($cntDone/$total)"
fi
done < <(tail -n "+$((last_lines + 1))" "$out_file")
last_lines=$curr_lines
fi
fi
sleep 2
done
# C. 最终校验
if (( cntDone != total )); then
echo "Warning: cntDone ($cntDone) != total ($total)"
fi
else
echo "Could not parse Job ID from output. Monitor skipped."
fi