From 930af071cc4ea8e0ef7c01c75a0ef46a131ee001 Mon Sep 17 00:00:00 2001 From: twq110 Date: Thu, 13 Aug 2026 18:10:17 +0900 Subject: [PATCH] =?UTF-8?q?[AI]=20[FIX]:=20SEC=20Form=204=20=EB=B0=8F=20AI?= =?UTF-8?q?=20=EC=9E=91=EC=97=85=20=EC=9A=B4=EC=98=81=20=EC=95=88=EC=A0=95?= =?UTF-8?q?=ED=99=94=20(#382)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 52 +++++++++++----- .gitignore | 1 + AI/modules/data_collector/README.md | 28 ++++++++- .../components/sec_edgar_parser.py | 30 ++++++++-- .../scripts/collect_sec_edgar.py | 2 + AI/tests/verify_sec_edgar_data.py | 59 +++++++++++++++++++ docker-compose.yml | 8 ++- 7 files changed, 157 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ecfef248..f208ccce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -44,6 +44,7 @@ jobs: - 'frontend/**' compose: - 'docker-compose.yml' + - '.github/workflows/deploy.yml' # 2. 백엔드 빌드 build-back: needs: [changes] # deploy는 job 들이 병렬 처리가 되므로, 리스트 항목이 모두 끝나야 시작된다는 조건 추가 @@ -200,23 +201,46 @@ jobs: cd ~/app/sisc-web + # AI 비밀값은 배포용 .env와 분리해 서버에서 유지합니다. + if [ -f .env.ai ]; then + chmod 600 .env.ai + else + echo "경고: .env.ai가 없어 SEC 수집 작업은 설정 전까지 실행할 수 없습니다." + fi + # GHCR 로그인 (백/프론트 둘 다 같은 레지스트리 사용) echo "${{ secrets.GHCR_READ_TOKEN }}" | docker login ghcr.io -u ${{ secrets.GHCR_READ_USER }} --password-stdin - # 실행 - docker compose pull api web redis npm # 최신 이미지 받기 (backend, frontend만) - docker compose up -d --remove-orphans api web redis npm db # 컨테이너 재기동 (backend, frontend만) - docker image prune -a -f + # 서비스 이미지와 예약 실행용 XAI 이미지를 함께 갱신합니다. + docker compose --profile jobs pull api web redis npm ai-sec ai-news ai-event-news + docker compose up -d --remove-orphans api web redis npm db + + # 실행 중이지 않은 XAI 이미지까지 지워지지 않도록 dangling 이미지만 정리합니다. + docker image prune -f - # 백엔드 헬스체크 (서비스 이름이 api일 때) - if docker ps --format '{{.Names}}' | grep -q "^api$"; then - echo "Waiting for api to be healthy..." - for i in {1..30}; do - status=$(docker inspect --format='{{json .State.Health.Status}}' api 2>/dev/null || echo '"none"') - if echo "$status" | grep -q healthy; then - echo "=== 서버 정상 작동 확인 ==="; break - fi - sleep 2 - done + # API가 없거나 제한 시간 안에 healthy가 되지 않으면 배포를 실패 처리합니다. + if ! docker ps --format '{{.Names}}' | grep -qx "api"; then + echo "api 컨테이너가 실행 중이 아닙니다." + docker compose logs --tail 200 api || true + exit 1 + fi + + echo "Waiting for api to be healthy..." + api_healthy=false + for i in {1..30}; do + health_body=$(curl --silent --show-error --fail --max-time 2 \ + http://127.0.0.1:8080/actuator/health 2>/dev/null || true) + if echo "$health_body" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"UP"'; then + api_healthy=true + echo "=== 서버 정상 작동 확인 ===" + break + fi + sleep 2 + done + + if [ "$api_healthy" != "true" ]; then + echo "api가 제한 시간 안에 healthy 상태가 되지 않았습니다." + docker compose logs --tail 200 api || true + exit 1 fi # 도커 용량 최적화 diff --git a/.gitignore b/.gitignore index 62f0ae9c..8a3b7f68 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .DS_Store # 환경 변수 & 민감 정보 .env +.env.ai *.secret *.key *.pem diff --git a/AI/modules/data_collector/README.md b/AI/modules/data_collector/README.md index 826478a2..e7d87aa5 100644 --- a/AI/modules/data_collector/README.md +++ b/AI/modules/data_collector/README.md @@ -118,6 +118,21 @@ Bash: export SEC_USER_AGENT="SISC Event Alpha Lab contact@example.com" ``` +서버에서는 자동 배포가 갱신하는 `.env`와 AI 전용 설정을 분리합니다. +저장소에 커밋하지 않는 `.env.ai`를 한 번 생성하면 이후 웹 배포에서도 유지됩니다. + +```bash +cd ~/app/sisc-web +install -m 600 /dev/null .env.ai +printf '%s\n' \ + 'SEC_USER_AGENT=SISC Event Alpha Lab contact@example.com' \ + > .env.ai +``` + +`.env.ai`는 Compose에서 선택적으로 읽으므로 일반 웹 서비스 배포에는 영향을 주지 +않습니다. 다만 `ai-sec`를 실행하기 전에는 연락 가능한 이메일을 반드시 설정해야 +합니다. + ### 실행 예시 AAPL의 8-K 실적 공시와 Form 4를 파일과 DB에 저장: @@ -151,13 +166,22 @@ python AI/modules/data_collector/scripts/collect_sec_edgar.py \ ``` 서버에서는 원문·캐시·로그를 `/mnt/storage/sec-edgar`에 보존하는 -Compose one-shot 작을 실행합니다. +Compose one-shot 작업을 실행합니다. + +```bash +docker compose --profile jobs run --rm ai-sec +``` + +운영 smoke test는 공시, 뉴스, 연결 작업 순서로 실행합니다. ```bash +docker compose --profile jobs pull ai-sec ai-news ai-event-news docker compose --profile jobs run --rm ai-sec +docker compose --profile jobs run --rm ai-news +docker compose --profile jobs run --rm ai-event-news ``` -최초 5년 백필은 정기 작과 분리해 한 번만 실행합니다. +최초 5년 백필은 정기 작업과 분리해 한 번만 실행합니다. ```bash docker compose --profile jobs run --rm ai-sec \ diff --git a/AI/modules/data_collector/components/sec_edgar_parser.py b/AI/modules/data_collector/components/sec_edgar_parser.py index 5b7a5b23..f0559bc2 100644 --- a/AI/modules/data_collector/components/sec_edgar_parser.py +++ b/AI/modules/data_collector/components/sec_edgar_parser.py @@ -185,6 +185,7 @@ def parse_filing_index( soup = BeautifulSoup(html, "html.parser") documents: list[SecDocument] = [] tables = soup.select("table.tableFile, table[summary*='Document Format Files']") + primary_name = primary_document.replace("\\", "/").rsplit("/", 1)[-1] for table in tables: for row in table.select("tr"): @@ -205,7 +206,13 @@ def parse_filing_index( document_name = link.get_text(" ", strip=True) or link["href"].rsplit("/", 1)[-1] document_type = cells[3].get_text(" ", strip=True) source_url = _raw_document_url(index_url, link["href"]) - is_primary = document_name.lower() == primary_document.lower() + # Form 4의 primaryDocument는 xslF345X*/ownership.xml처럼 XSL 변환 + # 경로를 포함하지만, 인덱스에는 변환본과 원본 XML이 함께 노출됩니다. + # 경로 전체가 아닌 파일명을 비교하되 XSL 경로는 원본으로 선택하지 않습니다. + is_primary = ( + document_name.lower() == primary_name.lower() + and not _is_xsl_transformed_url(source_url) + ) is_exhibit = document_type.upper().startswith("EX-") documents.append( @@ -226,10 +233,10 @@ def parse_filing_index( 0, SecDocument( sequence=1, - document_name=primary_document, + document_name=primary_name, document_type="PRIMARY", description="Primary document", - source_url=urljoin(base_url, primary_document), + source_url=urljoin(base_url, primary_name), is_primary=True, ), ) @@ -335,16 +342,27 @@ def _raw_document_url(index_url: str, href: str) -> str: def _deduplicate_documents(documents: list[SecDocument]) -> list[SecDocument]: result: list[SecDocument] = [] - seen: set[str] = set() + positions: dict[str, int] = {} for document in documents: key = document.document_name.lower() - if key in seen: + position = positions.get(key) + if position is not None: + if document.is_primary and not result[position].is_primary: + result[position] = document continue - seen.add(key) + positions[key] = len(result) result.append(document) return result +def _is_xsl_transformed_url(url: str) -> bool: + return any( + segment.lower().startswith("xsl") + for segment in urlparse(url).path.split("/") + if segment + ) + + def _nullable_text(value: Any) -> str | None: if value is None: return None diff --git a/AI/modules/data_collector/scripts/collect_sec_edgar.py b/AI/modules/data_collector/scripts/collect_sec_edgar.py index b3e8b1b7..b2ef0545 100644 --- a/AI/modules/data_collector/scripts/collect_sec_edgar.py +++ b/AI/modules/data_collector/scripts/collect_sec_edgar.py @@ -146,6 +146,8 @@ def main(argv: list[str] | None = None) -> None: limit=args.limit, ) print(f"[SEC EDGAR 수집기] 수집 완료: {stats}") + if stats["failed"]: + raise SystemExit(1) def _positive_int(value: str) -> int: diff --git a/AI/tests/verify_sec_edgar_data.py b/AI/tests/verify_sec_edgar_data.py index 0d9f04a4..d73dbb0d 100644 --- a/AI/tests/verify_sec_edgar_data.py +++ b/AI/tests/verify_sec_edgar_data.py @@ -39,6 +39,7 @@ ) from AI.modules.data_collector.scripts.collect_sec_edgar import ( _load_universe_tickers, + main as collect_cli_main, parse_args as parse_collect_args, ) from AI.modules.data_collector.scripts.query_sec_filings import main as query_cli_main @@ -70,6 +71,31 @@ def test_start와_lookback_days는_함께_사용할_수_없다(self): ["--tickers", "AAPL", "--start", "2026-08-01", "--lookback-days", "7"] ) + @patch( + "AI.modules.data_collector.scripts.collect_sec_edgar.SecEdgarDataCollector" + ) + def test_일부_공시_실패시_비정상_종료한다(self, collector_class): + collector_class.return_value.__enter__.return_value.collect.return_value = { + "companies": 1, + "filings": 1, + "documents": 1, + "transactions": 0, + "failed": 1, + } + + with self.assertRaises(SystemExit) as raised: + collect_cli_main( + [ + "--tickers", + "AAPL", + "--user-agent", + "SISC Test test@example.com", + "--recent-only", + ] + ) + + self.assertEqual(1, raised.exception.code) + class FakeSecClient: def __init__(self, *, json_payloads=None, responses=None): @@ -142,6 +168,39 @@ def test_공시_인덱스에서_원문과_99_1을_찾는다(self): self.assertTrue(documents[1].is_exhibit) self.assertEqual("EX-99.1", documents[1].document_type) + def test_form4는_xsl_html이_아닌_원본_xml을_primary로_선택한다(self): + html = """ + + + + + + + + + + + +
1FORM 4ownership.xml4
1FORM 4ownership.xml4
+ """ + + documents = parse_filing_index( + html, + index_url=( + "https://www.sec.gov/Archives/edgar/data/1343600/0001/" + "0001343600-26-000011-index.html" + ), + primary_document="xslF345X06/ownership.xml", + ) + + primary = [document for document in documents if document.is_primary] + self.assertEqual(1, len(primary)) + self.assertEqual("ownership.xml", primary[0].document_name) + self.assertEqual( + "https://www.sec.gov/Archives/edgar/data/707549/0001/ownership.xml", + primary[0].source_url, + ) + def test_html을_스크립트가_제거된_평문으로_바꾼다(self): content = (FIXTURE_DIR / "eight_k.html").read_bytes() diff --git a/docker-compose.yml b/docker-compose.yml index 4845b5bd..70fba6df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,6 +103,9 @@ services: image: ghcr.io/sisc-it/sisc-web-xai:${TAG:-latest} env_file: - .env + # 배포가 덮어쓰지 않는 AI 전용 비밀값 파일입니다. + - path: .env.ai + required: false environment: - TZ=UTC - DB_HOST=db @@ -110,7 +113,6 @@ services: - DB_NAME=sisc_db - DB_USER=${DB_USERNAME} - DB_PASSWORD=${DB_PASSWORD} - - SEC_USER_AGENT=${SEC_USER_AGENT} volumes: - /mnt/storage/sec-edgar:/mnt/sec-edgar command: @@ -142,6 +144,8 @@ services: image: ghcr.io/sisc-it/sisc-web-xai:${TAG:-latest} env_file: - .env + - path: .env.ai + required: false environment: - TZ=UTC - DB_HOST=db @@ -167,6 +171,8 @@ services: image: ghcr.io/sisc-it/sisc-web-xai:${TAG:-latest} env_file: - .env + - path: .env.ai + required: false environment: - TZ=UTC - DB_HOST=db