From 8950db9b76f0053cc8f890e8eab87d2ef4eb1c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cle=CC=81ment=20Doumouro?= Date: Wed, 18 Nov 2026 19:41:11 +0100 Subject: [PATCH] feature(passport-worker): skip pages using liteparse --- datashare-python/datashare_python/utils.py | 2 +- .../passport_worker/activities.py | 14 ++- .../passport-worker/passport_worker/config.py | 44 ++++++- .../passport_worker/liteparse_.py | 110 ++++++++++++++++++ .../passport_worker/objects.py | 65 ++++++++++- .../passport_worker/preprocessing.py | 79 +++++++++---- .../passport_worker/workflows.py | 8 +- workers/passport-worker/pyproject.toml | 15 ++- workers/passport-worker/tests/conftest.py | 24 ++++ .../tests/resources/liteparse_test.pdf | Bin 0 -> 32169 bytes .../passport-worker/tests/test_inference.py | 17 --- .../passport-worker/tests/test_liteparse.py | 99 ++++++++++++++++ .../tests/test_preprocessing.py | 17 +-- workers/passport-worker/uv.dist.lock | 31 +++-- workers/passport-worker/uv.lock | 31 +++-- 15 files changed, 483 insertions(+), 73 deletions(-) create mode 100644 workers/passport-worker/passport_worker/liteparse_.py create mode 100644 workers/passport-worker/tests/resources/liteparse_test.pdf create mode 100644 workers/passport-worker/tests/test_liteparse.py diff --git a/datashare-python/datashare_python/utils.py b/datashare-python/datashare_python/utils.py index 0fef1948..5cfb7de7 100644 --- a/datashare-python/datashare_python/utils.py +++ b/datashare-python/datashare_python/utils.py @@ -1111,7 +1111,7 @@ async def run_with_concurrency( ) -> AsyncGenerator: max_concurrency = asyncio.Semaphore(max_concurrency) aws = (_run_with_semaphore(aw, max_concurrency) for aw in aws) - async for res in asyncio.as_completed(aws): + for res in asyncio.as_completed(aws): yield await res diff --git a/workers/passport-worker/passport_worker/activities.py b/workers/passport-worker/passport_worker/activities.py index c08ca16b..ee776f1e 100644 --- a/workers/passport-worker/passport_worker/activities.py +++ b/workers/passport-worker/passport_worker/activities.py @@ -41,12 +41,14 @@ PassportDetectionResponse, PassportExportArgs, PassportExportResponse, + PDFPreprocessorConfig, PreprocessingBatches, PreprocessingConfig, ) from .preprocessing import ( ImagePreprocessor, PDFConverter, + PDFPreprocessor, convert_to_pdfs_act, preprocess_images_act, preprocess_pdfs_act, @@ -177,7 +179,7 @@ async def convert_to_pdfs( ] = None, ) -> tuple[Path, Path]: worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) - config = worker_config.preprocessing.pdfs + config = worker_config.preprocessing.pdfs.conversion cache = lifespan_pdf_converter_cache() cache_key = config_cache_key(config.pdf_converter) pdf_converter_factory = async_enter_cm( @@ -215,6 +217,7 @@ async def preprocess_pdfs( self, batch: Path, project: str, + config: PDFPreprocessorConfig, *, force_reprocessing: bool, progress: Annotated[ @@ -222,12 +225,19 @@ async def preprocess_pdfs( ] = None, ) -> tuple[Path, Path]: worker_config = cast(PassportWorkerConfig, lifespan_worker_config()) + worker_pdf_config = worker_config.preprocessing.pdfs.preprocessing workdir = worker_config.paths.workdir + logger.info("loading pdf preprocessor...") + # We don't cache the pdf preprocessor, it's lightweight + extras = worker_pdf_config.resolve_preprocessor_extras(config) + pdf_preprocessor = PDFPreprocessor.from_config(config, **extras) # We cache processing at the config level, we ignore package updates, # to discard the cache we just need to disable it in the args to overwrite - output_root = activity_workdir(workdir, project, caching_key="pdf-preprocessor") + result_cache_key = config_cache_key(config) + output_root = activity_workdir(workdir, project, caching_key=result_cache_key) output_root.mkdir(parents=True, exist_ok=True) successes, errors = await preprocess_pdfs_act( + pdf_preprocessor, batch, worker_config.paths, output_root=output_root, diff --git a/workers/passport-worker/passport_worker/config.py b/workers/passport-worker/passport_worker/config.py index 1d5e2f60..7ebd4989 100644 --- a/workers/passport-worker/passport_worker/config.py +++ b/workers/passport-worker/passport_worker/config.py @@ -1,7 +1,8 @@ +import os from concurrent.futures import ProcessPoolExecutor from datetime import timedelta from enum import StrEnum -from typing import ClassVar +from typing import Any, ClassVar import datashare_python from datashare_python.config import ( @@ -15,6 +16,8 @@ from icij_common.registrable import RegistrableConfig from pydantic import Field +from .objects import DefaultPDFPreprocessorConfig, PDFPreprocessorConfigBase + _ALL_LOGGERS = [datashare_python.__name__, __name__, "__main__"] _DEFAULT_LOGGERS = { @@ -68,13 +71,50 @@ class PDFConversionWorkerConfig(DatashareModel): ) +class LiteparsePDFPreprocessorWorkerConfig(BaseModel): + concurrency: int = Field(default_factory=os.cpu_count) + page_batch_size: int = 8 + + +class PDFPreprocessingWorkerConfig(BaseModel): + liteparse: LiteparsePDFPreprocessorWorkerConfig = Field( + default_factory=LiteparsePDFPreprocessorWorkerConfig + ) + + def resolve_preprocessor_extras( + self, preprocessor_config: PDFPreprocessorConfigBase + ) -> dict[str, Any]: + from .liteparse_ import LiteparsePDFPreprocessorConfig # noqa: PLC0415 + + match preprocessor_config: + case DefaultPDFPreprocessorConfig(): + return dict() + case LiteparsePDFPreprocessorConfig(): + return self.liteparse.model_dump() + raise TypeError(f"unexpected type: {preprocessor_config.__class__.__name__}") + + +class PDFProcessingWorkerConfig(BaseModel): + conversion: PDFConversionWorkerConfig = Field( + default_factory=PDFConversionWorkerConfig + ) + preprocessing: PDFPreprocessingWorkerConfig = Field( + default_factory=PDFPreprocessingWorkerConfig + ) + + def resolve_pdf_preprocessor_extras( + self, preprocessor_config: PDFPreprocessorConfigBase + ) -> dict[str, Any]: + return self.preprocessing.resolve_preprocessor_extras(preprocessor_config) + + class PreprocessingWorkerConfig(DatashareModel): target_n_pages_per_batch: int = 200 images: ImagePreprocessingWorkerConfig = Field( default_factory=ImagePreprocessingWorkerConfig ) - pdfs: PDFConversionWorkerConfig = Field(default_factory=PDFConversionWorkerConfig) + pdfs: PDFProcessingWorkerConfig = Field(default_factory=PDFProcessingWorkerConfig) def to_image_preprocessing_executor(self) -> ProcessPoolExecutor: return self.images.to_image_preprocessing_executor() diff --git a/workers/passport-worker/passport_worker/liteparse_.py b/workers/passport-worker/passport_worker/liteparse_.py new file mode 100644 index 00000000..845e0b33 --- /dev/null +++ b/workers/passport-worker/passport_worker/liteparse_.py @@ -0,0 +1,110 @@ +import asyncio +import logging +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Self + +from datashare_python.utils import run_with_concurrency +from liteparse.parser import LiteParse +from liteparse.types import PageComplexityStats + +from .objects import ( + LayoutReason, + LiteparsePDFPreprocessorConfig, + OCRReason, + PDFPreprocessorType, +) +from .preprocessing import DefaultPDFPreprocessor, PDFPreprocessor + +logger = logging.getLogger(__name__) + + +@PDFPreprocessor.register(PDFPreprocessorType.LITEPARSE) +class LiteparsePDFPreprocessor(PDFPreprocessor[LiteparsePDFPreprocessorConfig]): + def __init__( + self, + config: LiteparsePDFPreprocessorConfig, + inner: DefaultPDFPreprocessor, + page_batch_size: int = 8, + concurrency: int | None = None, + ): + super().__init__(config) + self._inner = inner + self._page_batch_size = page_batch_size + self._concurrency = concurrency or os.cpu_count() + + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + complexities = await self._compute_complexities(path, force_reprocessing) + ocr_reasons = self._config.page_filter.accept_ocr_reasons + layout_reasons = self._config.page_filter.accept_layout_reasons + skip = set() + for i, c in enumerate(complexities): + might_be_passport = may_have_passport( + c, ocr_reasons=ocr_reasons, layout_reasons=layout_reasons + ) + if might_be_passport: + skip.add(i) + pages = await self._inner.process_pdf( + path, skip, output_dir=output_dir, force_reprocessing=force_reprocessing + ) + return pages + + async def _compute_complexities( + self, path: Path, n_pages: int + ) -> list["PageComplexityStats"]: + batches = ( + pages for pages in _chunk_docs(n_pages, chunk_size=self._page_batch_size) + ) + aws = (_compute_batch_complexities(path, batch) for batch in batches) + complexities = [] + async for res in run_with_concurrency(aws, self._concurrency): + complexities.extend(res) + complexities = sorted(complexities, key=lambda c: c.page_number) + return complexities + + @classmethod + def _from_config(cls, config: LiteparsePDFPreprocessorConfig, **extras) -> Self: # noqa: ARG003 + inner = DefaultPDFPreprocessor(config.preprocessor) + return cls(config, inner) + + +def may_have_passport( + complexity: PageComplexityStats, + ocr_reasons: set[OCRReason], + layout_reasons: set[LayoutReason], +) -> bool: + if any(r in ocr_reasons for r in complexity.reasons): + return True + if complexity.layout is not None: + return any(r in layout_reasons for r in complexity.layout.reasons) + return False + + +Range = tuple[int, int] + + +def _chunk_docs(n_pages: int, *, chunk_size: int) -> Iterator[Range]: + for i in range(0, n_pages, chunk_size): + start = i + end = min(start + n_pages, n_pages) + rng = (start, end) + yield rng + + +async def _compute_batch_complexities( + path: Path, rng: Range +) -> list[PageComplexityStats]: + target_pages = _liteparse_range(rng) + parser = LiteParse(ocr_enabled=False, target_pages=target_pages) + with parser: + comps = await asyncio.to_thread(parser.is_complex, path) + return comps + + +def _liteparse_range(rng: Range) -> str: + # page range is a Python range, liteparse indexes from 1 and upper bound + # is inclusive + return f"{rng[0] + 1}-{rng[1]}" diff --git a/workers/passport-worker/passport_worker/objects.py b/workers/passport-worker/passport_worker/objects.py index 99630071..61544aec 100644 --- a/workers/passport-worker/passport_worker/objects.py +++ b/workers/passport-worker/passport_worker/objects.py @@ -21,7 +21,7 @@ WorkerFile, WorkerPaths, ) -from icij_common.pydantic_utils import safe_copy +from icij_common.pydantic_utils import make_enum_discriminator, safe_copy, tagged_union from icij_common.registrable import RegistrableConfig from passport_service.constants import ( DEFAULT_DETECTION_THRESHOLD, @@ -30,7 +30,7 @@ DEFAULT_NMS_THRESHOLD, ) from passport_service.objects import Passport -from pydantic import Field +from pydantic import Discriminator, Field DocumentSearchQuery = dict[str, Any] DocId = str @@ -71,12 +71,73 @@ class DefaultImagePreprocessorConfig(ImagePreprocessorConfigBase): ImagePreprocessorConfig = DefaultImagePreprocessorConfig +class PDFPreprocessorType(StrEnum): + DEFAULT = "default" + LITEPARSE = "liteparse" + + +class PDFPreprocessorConfigBase(DatashareModel, RegistrableConfig): + registry_key: ClassVar[str] = Field(frozen=True, default="type") + type: ClassVar[PDFPreprocessorType] + + +class DefaultPDFPreprocessorConfig(PDFPreprocessorConfigBase): + type: ClassVar[PDFPreprocessorType] = Field( + frozen=True, default=PDFPreprocessorType.DEFAULT + ) + + +class OCRReason(StrEnum): + SCANNED = "scanned" + NO_TEXT = "no-text" + SPARSE_TEXT = "sparse-text" + EMBEDDED_IMAGES = "embedded-images" + GARBLED = "garbled" + VECTOR_TEXT = "vector-text" + ANNOTATION_TEXT = "annotation-text" + + +class LayoutReason(StrEnum): + MULTI_COLUMN = "multi-column" + TABLE_LIKELY = "table-likely" + DENSE_GRAPHICS = "dense-graphics" + + +def _all_ocr_reasons() -> set[OCRReason]: + return set(OCRReason) + + +class LiteparsePageFilter(BaseModel): + accept_ocr_reasons: set[OCRReason] = Field(default_factory=_all_ocr_reasons) + accept_layout_reasons: set[LayoutReason] = Field(default_factory=set) + + +class LiteparsePDFPreprocessorConfig(PDFPreprocessorConfigBase): + type: ClassVar[PDFPreprocessorType] = Field( + frozen=True, default=PDFPreprocessorType.LITEPARSE + ) + preprocessor: DefaultPDFPreprocessorConfig = Field( + default_factory=DefaultPDFPreprocessorConfig + ) + page_filter: LiteparsePageFilter = Field(default_factory=LiteparsePageFilter) + + +PDFPreprocessorConfig = tagged_union( + PDFPreprocessorConfigBase.__subclasses__(), lambda t: t.type.default +) +pdf_preprocessor_discriminator = make_enum_discriminator("type", PDFPreprocessorType) + + class PreprocessingConfig(DatashareModel): use_caching: bool = True images: ImagePreprocessorConfig = Field( default_factory=DefaultImagePreprocessorConfig ) + pdfs: PDFPreprocessorConfig = Field( + discriminator=Discriminator(pdf_preprocessor_discriminator), + default_factory=DefaultPDFPreprocessorConfig, + ) class PassportDetectorType(StrEnum): diff --git a/workers/passport-worker/passport_worker/preprocessing.py b/workers/passport-worker/passport_worker/preprocessing.py index 67069d64..f4132a6f 100644 --- a/workers/passport-worker/passport_worker/preprocessing.py +++ b/workers/passport-worker/passport_worker/preprocessing.py @@ -1,11 +1,13 @@ import asyncio import logging from abc import abstractmethod +from collections.abc import AsyncGenerator from concurrent.futures import ProcessPoolExecutor +from contextlib import asynccontextmanager from functools import partial from pathlib import Path from types import TracebackType -from typing import Protocol, Self, TypeVar +from typing import Self, TypeVar from aiofile import async_open from aiohttp import ClientResponseError @@ -22,13 +24,14 @@ from icij_common.registrable import RegistrableFromConfig from passport_service import GotenbergClient from passport_service.constants import Colorspace -from passport_service.core import process_image, process_pdf +from passport_service.core import async_process_pdf, process_image from passport_service.exceptions import ( REPORTED_ERRORS, ProcessingTimeout, UnsupportedDocExtension, ) from passport_service.utils import run_with_concurrency +from pymupdf import Document, pymupdf from passport_worker.config import GotenbergPDFConverterConfig, PDFConverterType from passport_worker.constants import pil_supported_extensions @@ -36,10 +39,13 @@ DefaultImagePreprocessorConfig, ImagePreprocessorType, PassportProcessingError, + PDFPreprocessorType, ProcessedPage, WorkerPaths, ) +from .config import DefaultPDFPreprocessorConfig, PDFPreprocessorConfigBase + logger = logging.getLogger(__name__) @@ -113,17 +119,47 @@ def _from_config(cls, config: GotenbergPDFConverterConfig, **extras) -> Self: # return cls(config) -class PDFPreprocessor(Protocol): - def __call__( - self, - pdf_path: Path, - pdf_bytes: bytes, - output_dir: Path, - *, - force_reprocessing: bool, +class PDFPreprocessor[C: PDFPreprocessorConfigBase](RegistrableFromConfig): + def __init__(self, config: C): + self._config = config + + @abstractmethod + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool ) -> list[Path]: ... +@PDFPreprocessor.register(PDFPreprocessorType.DEFAULT) +class DefaultPDFPreprocessor(PDFPreprocessor): + async def __call__( + self, path: Path, *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + pages = await self.process_pdf( + path, + output_dir=output_dir, + force_reprocessing=force_reprocessing, + skip=set(), + ) + return pages + + async def process_pdf( + self, path: Path, skip: set[int], *, output_dir: Path, force_reprocessing: bool + ) -> list[Path]: + async with async_open_pdf(path) as pdf_doc: + return await async_process_pdf( + pdf_doc, + path, + output_dir=output_dir, + force_reprocessing=force_reprocessing, + colorspace=Colorspace.RGB, + skip=skip, + ) + + @classmethod + def _from_config(cls, config: DefaultPDFPreprocessorConfig, **extras) -> Self: # noqa: ARG003 + return cls(config) + + def preprocess_images_act( batch: Path, paths: WorkerPaths, @@ -235,16 +271,14 @@ def _preprocess_image_doc( async def preprocess_pdfs_act( + pdf_preprocessor: PDFPreprocessor, batch: Path, paths: WorkerPaths, - pdf_preprocessor: PDFPreprocessor | None = None, *, output_root: Path, force_reprocessing: bool, progress: AsyncProgressRateHandler | None = None, ) -> tuple[list[ProcessedPage], list[PassportProcessingError]]: - if pdf_preprocessor is None: - pdf_preprocessor = partial(process_pdf, colorspace=Colorspace.RGB) docs = [d async for d in async_read_jsonl_as(batch, PROCESSED_FILE_TA)] n_docs = len(docs) if progress is not None: @@ -303,14 +337,10 @@ async def _preprocess_pdf( output_root: Path, ) -> list[ProcessedPage]: pdf_path = doc.locate(paths) - async with async_open(pdf_path, "rb") as f: - pdf_bytes = await f.read() output_dir = output_root / safe_dir(doc.doc_id) / doc.doc_id output_dir.mkdir(parents=True, exist_ok=True) - pages = await asyncio.to_thread( - pdf_processor, + pages = await pdf_processor( pdf_path, - pdf_bytes, output_dir=output_dir, force_reprocessing=force_reprocessing, ) @@ -322,8 +352,6 @@ async def _preprocess_pdf( async def is_valid_pdf(path: Path) -> bool: - import pymupdf # noqa: PLC0415 - if not path.exists(): return False async with async_open(path, "rb") as f: @@ -338,3 +366,14 @@ async def is_valid_pdf(path: Path) -> bool: finally: if doc is not None: doc.close() + + +@asynccontextmanager +async def async_open_pdf(path: Path) -> AsyncGenerator[Document]: + doc = None + try: + doc = await asyncio.to_thread(pymupdf.open, path) + yield doc + finally: + if doc is not None: + doc.close() diff --git a/workers/passport-worker/passport_worker/workflows.py b/workers/passport-worker/passport_worker/workflows.py index ed2ebe1f..637034ac 100644 --- a/workers/passport-worker/passport_worker/workflows.py +++ b/workers/passport-worker/passport_worker/workflows.py @@ -166,7 +166,7 @@ async def preprocess( pdf_batches = preprocessing_batches.pdfs + pdf_paths preprocess_pdfs_tasks = _process_pdfs_tasks( pdf_batches, - args.project, + args, force_reprocessing=force_reprocessing, worker_config=worker_config, ) @@ -225,17 +225,19 @@ def _convert_to_pdfs_tasks( def _process_pdfs_tasks( batches: Batches, - project: str, + args: PassportDetectionArgs, *, force_reprocessing: bool, worker_config: PassportWorkerConfig, ) -> list[Coroutine]: all_tasks = [] + config = args.config.preprocessing.pdfs + project = args.project for b in batches: all_tasks.append( execute_activity( PassportDetectionActivities.preprocess_pdfs, - args=(b, project, force_reprocessing), + args=(b, project, config, force_reprocessing), task_queue=TaskQueue.IO, timeouts=worker_config.timeouts.pdf_preprocessing, ) diff --git a/workers/passport-worker/pyproject.toml b/workers/passport-worker/pyproject.toml index ca37fe21..8250aca5 100644 --- a/workers/passport-worker/pyproject.toml +++ b/workers/passport-worker/pyproject.toml @@ -11,19 +11,24 @@ requires-python = ">=3.12,<3.14" dependencies = [ "datashare-python~=0.12.12", "icij-common[elasticsearch-8]~=0.9.3", - "icij-passport-core==0.11.9", + "icij-passport-core==0.11.13", + # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + # The LiteparsePDFPreprocessor was benched for this specific version + # bumping it might result in passport false negative + "liteparse==2.19.0", + # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ ] [project.optional-dependencies] preprocessing = [ - "icij-passport-core[preprocessing]==0.11.9", + "icij-passport-core[preprocessing]==0.11.13", "pandas~=3.0", ] inference-gpu = [ - "icij-passport-core[inference,gpu]==0.11.9" + "icij-passport-core[inference,gpu]==0.11.13" ] inference-cpu = [ - "icij-passport-core[inference,cpu]==0.11.9" + "icij-passport-core[inference,cpu]==0.11.13" ] [tool.uv.sources] @@ -85,4 +90,4 @@ log_file_date_format = "%Y-%m-%d %H:%M:%S" [tool.ruff] target-version = "py312" -extend = "../../qa/ruff.toml" \ No newline at end of file +extend = "../../qa/ruff.toml" diff --git a/workers/passport-worker/tests/conftest.py b/workers/passport-worker/tests/conftest.py index a3ca091b..8f2d5143 100644 --- a/workers/passport-worker/tests/conftest.py +++ b/workers/passport-worker/tests/conftest.py @@ -1,6 +1,7 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path +from typing import Self import pytest from _pytest.tmpdir import TempPathFactory @@ -39,6 +40,7 @@ from datashare_python.utils import artifacts_dir, ext_to_mime_types from icij_common.es import ESClient from icij_common.pydantic_utils import safe_copy +from icij_common.registrable import RegistrableConfig from passport_worker.activities import Activity from passport_worker.config import ( @@ -46,6 +48,7 @@ PassportWorkerConfig, PreprocessingWorkerConfig, ) +from passport_worker.preprocessing import PDFPreprocessor from passport_worker.workflows import TaskQueue from . import DOCS_PATH, TEST_RESOURCE_PATH @@ -434,3 +437,24 @@ def test_model_path() -> Path: if not test_model_path.exists(): raise ValueError("place a model in here or implement automatic model DL") return test_model_path + + +class MockPDFPreprocessor(PDFPreprocessor): + def __init__(self, res: list[list[Path] | Exception]): + self._res = iter(res) + + async def __call__( + self, + path: Path, # noqa: ARG002 + *, + output_dir: Path, # noqa: ARG002 + force_reprocessing: bool, # noqa: ARG002 + ) -> list[Path]: + r = next(self._res) + if isinstance(r, Exception): + raise r + return r + + @classmethod + def _from_config(cls, config: RegistrableConfig, **extras) -> Self: # noqa: ARG003 + return cls() diff --git a/workers/passport-worker/tests/resources/liteparse_test.pdf b/workers/passport-worker/tests/resources/liteparse_test.pdf new file mode 100644 index 0000000000000000000000000000000000000000..94012f042d43cd26be1aeff5ab823d117fa3f69e GIT binary patch literal 32169 zcmeFZ1yqz@w>YeVQc_|dAp;W9QbUJ=NJvO`cQ=wF1|cONIfSTyAR*l#A|<6X3?<#& zF?`PqA;$ZD_xFGA{nmHa`Y&B;9?qO+pM7@kea>tuSrJi27AAH~s`lCGADAql+aPN_ zGfX}{W(89R3q#N?MJoryg8|6ETHn#q(8_@plUdfz+Q3oY&<=D<-rQ7|82|yYWC7m> z-M6+j2Hp{|*0%@A8Cn?_+8Nr>3J72Vj|iY_KY{D(f>!9n zG*AdCe!&{{U2U(B63Wak2fBR$2Vcity^$A(OMtsd14LQS{JOmz3yngFKUveu+L6C# zE!iVn_*xr%HW#> z**l@*$*mVpaY#v6xr$7}k}c(IOmAN8Rw7T84~2sY#Kq*-&NKAtkXg^n{a zOy&DJ*Bv{Q$6@J{m(M0iODk41iI9?)7uo$lSE+)z2)=`$XOwiqkE&nxl2>g?w(XYH z9Ox$wTzCD1hIc<=;9M$p`uZgSf;nn;+AVe*{g<^*-ahrlXe0gD3)OReQGWZlr)L1%fYyXSi43l|6^ytl4SpE&v)CfzvM^JdBNKMBtHli&(2HijT(6&XDf^TCoiy4|4n>c{jxw$Zzg{>{D?c{BA^$kJl%;Lhr zcXjOnEkiI;P7_cKQwMulLpxz>OB-t|K*Du9JUly^|$&|xSqqA0--B5Eb z-`rsuhPcjNcz$>M@VS5;Y$3b>viZ5dlI1LO+rMdsuW!JlkqP-25DOd#}Bd*3Np_AX~$= z25p(iU}G7{8{&Fam1wPFtuiv90*UZPt<#C|+PRlNw_VQ{%=MQWPA)X_I#~`ByEO)B zx{PTr4wmG?*IUN2)SVVEJ*$4=Ok~Mzaqx<*nZ>QmolTr>*3Wn`+65L$${tvg}%29c(gEgh*c^OrnahD#h43GPsTPj7Bj}-4C~U3g}|m~$yA8@rSL$v z6}&7aY$B49{mjg%VMU7I)Xro{zSQa&{S%R?uGgI{HV&1ZX!jtFHqs)_xb{Fjg4#+m znRd6ft8O)G68tq?SP-(ebm2Pc*>i6tUkliCG1h=1?7mXCm1#vt8IeLINqkPx*Tdd# zmdy()$;MG>5l> z+dwzR!JyIkg5qfD4VmeH5r3WosYZb{?#Yq!a6ZSDyyv(w>RU_WoYIrx!;=JJ-4re0 zcJWAR>28Y5#gFYO-)JkzoN|l53>E-NwJ%ML)H&f6U1D()IpyZ7=4J*l+jB#%$-U_j z8}w=5#;oQ_@M^EW;6ZciT#3Qns}K(;l;3njzuju;TKlfYcm8?r0w~SZiSqo_Y?{XK z3c37^ZdKfzb2m3q^pjg3a2 zRHrHWynZVKNtnn=O;0XXZL+pTI>Gs-)}%np!rU- z8~N&mGR2!!cM@b}lvfRXR-ahx+_N9*v9L{HPr`=30630{F5-Fp5}=RlgSb$u&KRnA z#@+IQTj_C)6FY!12Pfx`EvO(q~J&D0W(ELJM&3-uOXYGtvEZ+ z9MpTpJbPR@zyRo0_1p(ws8N}HtCHL7Gb`CcZKkkBLlreA1rL4C*Y1eKZ~6MR+0ZBK zBKyErz9F>RVgE;Mdye&DVNAPkETpgTKE!^s37=)Un6afX3#&F>lWW2urqHOfXReA3 ztF}z#9SJzKx~oI?Dd!iHVDsc5t;{Zo@V)^_v{qf>uxG^feuc@ixRY3yCmS!Pg!isR zS80bF8EC-jUJ587DR_7zBP+KBe~UXggY1-2s=b+)@GYcapCCk3WdtfVk(j zXFslwv5RY>4PJz?#uZg4r;~u)p_oTz;nP&Cq{h%^4#_J}-qE}VI~mLj8Z!5Y<)6%6 zH>s#IS5wGX+>BAyzrVJ})*ZJu&sZ4zx^*YW*5$m@j~+udtD-Fetsz_Ex;Hgp6M{D5 zU(^@fW}Zia%XlHvY|7GQ94}zC1D>rT;pSQ@2QB^gBjR=gm1P!+-tIO~F=uIRZ$w0b zEj)*#Ba@jPO{GE6BI2_ai$3Uo9|^kWpH(zn7WTndU`YjlPOw-cXgyvR&!B8RF2NeP z#YvD{Ffh=$-!>C&ZLPXBwmNE6x=W+gXifc~+z>v5z~Txqn16T2yEKB&Lvxq(CRIYk zU`j-~TItGsGjsX=0!nKI*VHVjsD|5Yf56lVaf&urcjlEFnMRq}yg%_^`KI3_++$-e zp)4LUKSWenYB<3kELb|%*{wa$zdGXW9GR}hIax)e{2+E&zE!(m&rf+sW#6h*k{Haa zmGAUz5_fUuZPC1pyiu!*$#k3ItX6N{{P!zaE$+kF2}N+pI++G|WM#`&SYG*u-Bb?i z%)$QMj=n-8!J@HH+pt{omiVr0lVxB@bqU_KRcos`mApqv`^x}bAa2!pls?VWxW;TR zay9iN@sMgjNj*O5u9!AyArlY_?^V5srP($oa$|--T?e4!LqUO{3Nsar{&rf za040e(?X4m^6$Gpwu(1z@Ga{XhwkNTq&jA4v~NW;TE`CVPFKiLx$wn(i0*h0R;jeR zLB=mIy^z_pDpTzIFyel{eY{#vU>sI%sdi~ZbcGqzz(-79u#jm!vjd z&)rHkvu?~XnV2*b0U@-3d-wzox_1wudBCr5HrMZwsV;vt8}ZboC`b}#%k4Ji!De==yOb%dnC{0~qn8#7L>5(~hjGL)Z4?F`EBIc??8jzH zDyEYjuDJtsl&$r3xwvk?7&LX*#*h}l`~cIZ5ug<_GN+1$EEk@k4@KZGhgR)Y2`$d>{M%wFMq1_X@wIUktS}m zR@XqNEvdq~E`u-*-eD<5P(&+D3FH#sr^=vaTQg^Lq>Q&3XN zdNIB=L1n+zf0lEe-+{6ePW7P+rz{?8!+K98oUYR@soeZvdR%RtL_D9JM|e0yp#3R} z-pA6yxLu4ftTQ4MDl<#fQM;dJJUJmFos2rPvKBOZRY35^_dqtKNHw-^F8nEIl!?{c z5D&NwPvq9z>&b-jWkFBQ(Tb1;-D^So(9DUoFJc|K*Rs^05%10PjavLi3g~HCxEw|d ziq&}CM-0H`?5h17QmE9;5YTP}J-T3%!{I+tJDmZ!iAR3)yKZ!49m;DseGtT__(nTc1k(nqNq*46PaG zlp1oL5=&Luor6^qOfZ`8c}{Q9%4B%3@pf*fxm~*i*mlSHi21OE8)NI9Yul0IW|p63 zNzD+9o{-Pn;LqL(>{_j?bcS?Yg;!8hQrJb*xGusP387c1>{V%1k z`Ly@esB(bgbYi>B#y?)$g^40Khs>D*?ft?OA?l0wEE%J~2Nv&=Oh>j`SymHD(kwR= zElHINnk#ZL^zUw#_7Y4`!>00@;8H1=xa)fj60(e?&0u4q1baC5;@%{g5o3v1!aC{l z^m!K?UW(0XCUN^7l)A5h_W`Y?SKcjv?6{1E0Je)+d1qnDb4EMfasnmTFbr(mzbQ6a zB`6i`F{hBEk;>?nA3bcq0(h6+fNx!sDULmq%X@RfKh7364NWw)wjAsc6BFYyX%#1H zNXAd&(G70gcdP2vLv{T&L@oSlqzR7^f^y(uMfQwP#4dv1Y*$( zd+k>Uj43-OCGr(g0q(9R&SkW3%A50y6MydV&)62t&an5(9dK%*y0J}0cAPJ5g?H#e zKhRJ)w(j(O?wx<%OcmZXxkjznVT)VoXb77dlB$@GX-TSt@@%FxLlhzBkP!Ii*atp};ups1WVdX9)GoEe9e)pxGM_m!67imNBmxle1 zycK&T%k{Om&acus*B3qd9~snnWl46SKDevfWiTG`0UxA=SfkYbt3p zB~Jj}G2+5}Jg_sjJa)pcJxE2PM9JxfPE(0yfWdE@c6|>^0qhbzv;f;x(4x3&bKY6dfX$0p z19Fg-N5t#)^#${cKQqFFGB&)N-I}v&+(BaXo&_O)|Xiak#q?eE++J z^kN1ClD9nEDeeKL{lK`8@WIJRFuH$RWg5Ca9BR+}XuOs&Ng_u@K0>QAaz~u{bw!S; z`lxqZ=i)~CV1F*)4S1*Ngp(w+XW2)8+}m;0cDHMWW>vu2-OEiQ94f;7nJC+K#XpQ! z&4fPeFEDD{u*-KE-g2HSj>lRVTB@pxK8R_C!4?)u7&OB-Y4obm65icwt>931ZJ(4O zt(qNztUW50gDq7B-DxeDgaQj$HU{yd+^DqE>-!d7QP{c~IN3Z!Kbchx?jn)4^0t)H zTy?2Yy;BhXL#=)KjiR7tHb0bO2oGDMG zE1bV?v3=^Z;qJHP-xHkfFfZR8I%ypfToa<0qc4Y2t;EFnVmt(Y2AZ7o3Z=4 zVR>s2+fVzy>}=6zr5s3B7HpCrpeb@2)rdpx~9^o?6AVt>w@IA_*+AlBqPyxY0oarjZO?f!jFzTRB7#rE03 zl5xI54DBJ?$;dj%z*VyCh*&qI7rQ-l`%`D%*Fg0P`&nV?&h(w-Xu%hq1TF0L znaj3Y&VSZo8{L@OX$8s|p$W;eTppp+Y1@_`x1UJ6Y;%;efT0%dEP^Ia&Un@19%{)} z_(Fz!LJpKMtAoLB;!3N5Zj$Btv+T*av1^rOu3evLnQxdBtFF(*I>VP!R5vv{!MmI7 zi&5Xowh`!lPpM6+tDXzzXet~^6pyUGwYcF6owEEzm^yhK;oM6?# z#<`rW&C@yOUk1Y;&EQ(*=G+(9*=v+bT8s<6kbJ>~v_Q=t4TcTYCQnn_g~b{yexKgt zj~$~0C{)NfSqc#-dY6rDQul^RoG;3Ke=^nH`~F%Xw=>z^50TV9=dHS(S&ymZ+F8aV zENuS0&o$#88|Y`u=iK{R=f@pQNsC>Z*I>-G2Qv+%jYE%kiN(b>D;k_)#W5Emhq(6p z4>o}>T00XG7fwg$kYbP3N25+{4}lD@rM?tCuwWz*RfO=$arjS)!yah?JMPqvVrIit>6?38UM zaI?BR@#Y(&4~zp9;|XyvX-=a9BS?#DuB&5AtyU*^GR+||rLxImZG-W^8gLCwELQxG z%0RvL$su4qWLm>%SwIyBS{oHoOa_)>QH$hL%QbXMh2X$)jjdDr3fIEBb1JQ1`9LP) zp-7id+OpIW9j98cTF=crz`-_gLuNK*Q?Q&OHKC=PbKq8r^}RN{y?sU4e8Ex*xC zVp*5Vrzr(1n+5K=_)HG@Ryy@_1L<*=rQ70Gs+rT!a!!H<#dKC~&SWKIPJs%BXZ$Ht zLbwEuaL`UUa+zI9)(h9yDjR1Sd2!v@dNfAH%$Q?&G}~#X#RT~yxtPn6F)VVv$b+l6 z=qthCA#r&bp8or~=U#XC`ps*6hcQX3Yt3ynEX66eg| zDv))M5q-c*A@FOz61x&=?h!cR&XMAtyxjV|>PTMkT!blQmv(j43$&z+Hzv%_6-tjw zS;<=UWzk0}ESb`Wl+9iXd~Ge0sx{WFAf+o)=2Bqa@TO0Ti-~E$ z$T#dbtO`CQCdJ)&Qj5E$B}}$*Z#<^wy-cU6MyVHh9r6@Dcznz%ZkeysrlcQQfbR?! z&-QD%8)OwKWDEcaV3pRQx+?9e9i{*INsvFKiTa7m7jV2*KGNgcL@&z;$@D)FvU}aWoaW8i{ zNcL9TYsK~?aIJVPum5mlLMyveZL=#irEuos&An=>!{Z09dg76+H#pD<@8*5;l{RBn zH1kF<=Xz(vM8RN8Ocpp6Ef0a5&%sxu$8RKQA>Ze=R%R_Ql5L+3EXVZYLMKGfDg~EI)pwG(3kMHOoZwx7 zo)ZE)BE@QqojX_gYzMKe;OvGQOYTgix@xSG1!^7>>)1~#1{tc$>q)?+aOLemObuCgWVvjr`Ig9mN+60!jhS=l*cp})tkjs_-K)>m)rRy{$QaJ=ZW?Qm z*ua6DH%jy13-2PCyHpA}Q^Go+AaeOTGo?>k)d$PihA<1XESjwzTe^JAE|*q|C3PbB zS=gUaE@QIYXXeaAG^@N%o5I8*h(u}UCraTJo8LmJ1^1$;QYIzY<26=|lyd2ncRJ(0+8?J5}bC&5Y9HwxThKkGvk$(LQC=90);k^J0 zpuMwF!omiH^6Z;_o!gCfCGUm3?<*{UyolsGs0X!bZ*^eR$tbm!hdaLSh5-3&X$2f? zniZD9L*}*;x>hw5kg&3Z#cYfvXSrwda=;$2`?$0|ng{SYpa!;;`qe|GVky$JP?N`V zg=#ePy)(2c!#;;scXj&9c6oLnRlF0mVXrP{_Ho$No7sbEqQr0yyNWJL@RdBruw>0JxCweAM0c|ivXRnuai(6uxF z1Crp;T`bX1^2Kx)$B(CCe&HsKj3KR)xdAa7M}(bO)2+;`(N`0>>-YRt2lP92nRk?< zk6E$zV$;Vm`Wvi--khjmkW-wgchNMk<{1>jv;^oJI!U!X`IlRE!=P8`8!8wDLrm5@W5V%e;JR(;RqDkROv~EF1H~rWU!M*o)H@qy zP-1ix=RU|S0z8~gymojCX){5D1wk|*`3#W#c?^5B-jcP?`YZhYP9$4ArQUOUyg2W& zI8{yF^K@WjqEM(3(3TII8#j5(C3X6E)=$zze#mdF;CI)b{Cp>C^7B4(rR~m(^~JgE z37y$>W5aJ(W;yrOJsRUDR;^iKnjN2BI4XuEG^qt z$#_*=EMdDSM!}mOs*}3~6=qo2CNuhq#^1K$x@*D(N?S`brW@^gEV{cHB%2GK4D`zd z4#W(RLl-M^6-hTJJf9rLc&|U5yH--z@|vUFf~G6t{!T{CE2gTBe!LWTIU{cT^^nz8 zmDPPi0w0&n@mnu|()c!PZrlEb8jt0V>d8jlmcXIG+l)}$GN;B3tb<^ylF`d`{J8yHs1R?>rB_%X3Jj|(ey&8RWqjg*3QF?K6e21Bf;G& zsz7Yr`(F=+l7|MJWV(R^iDsZWkG*;o-nclkx!1BP0D07%Py>`Y`rW(t?a4Hqs>{w- z@*6*&N#IqR&Da*;UEi(_gc`J?WlXOP^)VfvWy^ka^dly4&UI6swfC4Ug&DMaKv&(` zXUsxh7}a#m)pk0l#~RSY0qLCR!}XC?!TE6g%HmXhu2E+2f>_t8cm#EP!BXPzwN)&? zydYJnnW#c7*-*Ho|97{2)wY*h2=#tKZ(S>*1Si zn%DVvdX<1BZXqszuA{FZaxZ|4eyN* z^l^c!$M3I&E;kPdi+uQWlU}PqX-hn^HiM&!^J>QIPDDFLi0IpfsdD%Z-wy}-9;4d6 z9z1Y40a$aE*3D=M>nh^x67R6OJJqNw!^v6Iu%O)B);c(GmAm0ub8ltyICRC(r(J;^ zhF74C56)`9ncU;#Uw&v{G`9^~8#HVb?1K#Nm+xvnnvJtBticBadgnVFg=|r92}sgJ z-_`~_BcL0fNSUNZ>eu5C7q%Zaf-E|2HCM~5(AUJoA$S~wk;HU?#GD-2Ft z{9!>5EBb?@LAagH3~{Wh`(sB=+Ql2n@RdDaRYs+afCO=Pf6;4BtJ`%Z#yTr3DLe3a z{0tTk)d6ZWvN|Cm;EWF(%KCC!jCyv~0sw2RjVldfhzj#@ROE^d#(vCQoNoGI)W-8= zwJzAA!xuPOmOH!)U__5wXpj@AZWyMPu@4+}MjTq$kV2GmZ=E4MmVpke2@9}#_SHU< za-F^$2b)>oq5-zU1p#BLw@t29lejHF97tFo1k~qmHS3-f>yHXFcK-E)8m0;Z=H4gc}WKdi7Al9yqDE8C?{7d5D&5pr{@84{>fz?Jl}U0>{L8X z@IXGip!#-rb!d%Q@By*RuegpYcf9QC^E03v$As6R2qq!$?$ilNXLb_i1bjtP0P%&myuE^dY3-UI0IoAi9~Mn}IaaU{Awd4(<_ zhr*pMaDoT&q2`jw;*8be436jZW0!I?prvY6{L`B@M@H!?0aBl)34M&=Y zq!k{5(*PHLYT}V}gYdR>9-L_65hFnnplg4j(C?H4r0h1JhqoL68*qw{$Y~)*7WGIh zkM|<{#gRh*o*=_&n!`^kW}?+c)Y93Z!bEl)n!rA~pM` z*+T%mi}4<~Lv-#}fX5bq0>$tcd%e^15Lb1OVsV<+2muF({5!AziLJo%b-zu0qKU^_ z2QhWvdHsGh@*mj>bdf{cj+uR|uoMHw;)qbMLkT}kp3@TURnIJZByK-w>g$2m3 zj&$LaDg8xfLbSb)mHwxNJW_?Zc|i+o>X1VbtTO@27JAIqwNz&qNcE z!QpSy@U8tyV;%25#M>wt2~+Z_4L(_zodc)@p%mGX~WpM(~_?iz8fft>rtUS9kj-h0*8-93l;3v~c&0NMPL zI%m&?bo^-|vr+p>OJJoXPqn>I3kASlU~v%Z&)^wA$m|92ws~1nqBuwjLrr{&VC14PsG-LGfppbS$!m z5_F>4|5q$dvHSlIVuAiI@-GUq0E|->=*XjDafqBsjgJNWk9oJEkeQ$FUo!1e@dBWW zKUF(~>{RxCq}nG@@na|c`QwFsf0U8`)7WALCFFF@iTdA^ zo&urT72szJ2rp3cf0~{G$;B;--_uiMbnzEU^|$o&pM{)HIqy&&8imt5Jnb1yS4({N zKtKKQvEM$Gp#GnF5g=svi|<5W^m~2^c!uETf8?hKwfQN$M?n_w`rq@@e-TpoQXZC( zevbqGl70y>+&TQs4q~ZaMf@}w4_$@qk$U`6mJ=GAKU(LQj0p8OD)pTv;~|WHkd%ku z*8bsS(Z4(=qWCv;l@q33et;~~i2PZ8ImYd1>WS+*xh;j1`{R)KSp9!3zx*{hJt+$Q zF5iDm$$?db{!3{HqQ?3=M5Le#QTl}6IMUKXNkIbqYwq&%pVMslC*A|0`^ft72_+Ay2>7GZ!5MPjzU~Z^e`MK!hySaxf^ID+ga;ogzOLFmxfQ*g)q%8q{qWTl!VZ!%|7M!w-|Jxh~x`>_1^l?~o zxJjfZ!W{UsGA%@T${Cz8Pk7T8PAGQbv5spl2+94M$NE`$#k)>-l!g78N&PkBI*eWa zU^L#gtO$VpuM9-w0)Ht~{52KFyGD0M>0j65fA<@IDeoUk>mezAt_xau`crz2_25*R z@{6MW&0C)M;NSEBZ(9rr_*Xpu$o(rBfjR#uAw{L&PG37R+edaD3Gi=OI4Ja#FF0kL z{!wuT>8uc9a$@8EMRbYhTSk89&rb9CFe>{=?;}n9C()%4D3s{WWFkVyP1zavztniT z2LK=m$4opn%8!L~59@`#r-H`gX{Un5(-G7^u@fNVEmh>cPDlj8lYEIZ*X~M(PX$tgH*74}@?6h?}bqq-%cxvu3uTRbW#RB}tXMul}iobeH zz_w}pawZRu5*f+;uZky*4`BbJtm9v#TL1be5OMH(N&qg4{?qf}f0B3rkt1-`1i9R4 zD|J*C|6Lva>rv)k&xik20{%~;%>Ry~z@PN}Z?zSFEzBUUVEiqd`FjHX|CLmDZ9NgV z3Wo^L<_G=Rmx%jkU!s54E{K(t?R2vsk6=mj>f3nF7I*0vc>qx@#+^zG?YUA~y@Xx;NvGzvZ{gKy5oNjWk<-p()Oz*hL*cbi_uF`V zmg5W(xE=@MM&UZVg9i)et$3%31wLX(&<1`n?IMtvxn5B|tMiS0HZ#eQx;*1kIHn!N zN(gH1w+m36IOXqscrR<*YovP#HODWlV43L$n744`P-FTCgFGc-?Vc-+8F z{SXJxAx1yHKV{dBRTiNX3U-D7VWk}{0T$l|u^^=o!~$YDY@>FHvZosxA{FCsBZ&9q ztgV5zZ2%*V*^E>z#5dA_(n}h;Af)I}-w=Z!7No5Hgw2Z3!C$B?tm~j_VQq|f%NVdG zhe`@)y{?_PxRsGLLS+s&aXfPP9nk&_(04-s@{!^nV+Ue6BCV3Cp|hc#tev3|&|1;z z$Wk4G`r|81W)V|+8w*_*fJcYD9FKuN(6urK-7>T~)tZs@6nzdmEFy@=ew+Q@cb(&A z=lZ3W;Jh1#yF%xc&4a;BqlAXCXiBy)gZeUq(8mw%f-odORLsOWH*?N60gVuS8J;kl zJ%7&^ivmAV&HQ=6h@5(cr3(f=jfo^(5jpy;ktaez3K|q*`1hn1HcOqo(K|gW;R5>+ zaDhbw_+-C*3-4xyeJlTBGZX{;@uPF~Q-V2VdZ^zUM$a^#x$Q94!$R;9onGk0nV$R7 zQ7UHprKXHR@n^q5nx6HSCB>(No>9Yk?IgE7HIu#Bv`s?De${4x*F$UI91*kOxjfQO z0n^bP;^gQV8nOkg=Org+CFWsE*kOeR#D?&2T0 z(I?`3&imqHh8SLmt7e~XK>6b@tw7H=N6Bl16hWeo6;NceGNHvdh*JZnJP&8(tO39>(!%6JU9ptH;SXfA!E28zmdO9K>e=TZq#>|K0MG=f`~$Mi0N z4th7735Cf0^gp@U9}Kml>ktik(0;w_p1=HI_A+_#Ybn3LA>4hP_v#9q*fVQ}{u6iO zir>TO-sjGwFh1cav|8?ztz5?tZrka$idpYhF#bL{aXA-HaL;P7BDR9hsb?Y!?9{BF zQ-JbuEufw@a!FM8MFXX6#M|4LsQR}rppM?Yfg1UP6;I;^G=XwJbiwugv*37#!%hve(zK8o9vqe zdJknD2W0a>UZWSkaBZq_XQ*Z6s|Wf(X4dL4j9=PVaLJk;@?*31JFqSnTRESlX>e_T zyxenviu0bfcVM!}8R`cUAt8)sCSJ4D5~$cp1*G^^rxP;Y5l$dvPB7LNVja;d!?!&n z@ew`up-(YkAl-wz{8tmt=CY9Sjb3;|cHy<)%xyKXH_O_OUEdcGuFCpEM+JUhA)vE+ z9X}~m^Qc3gIzoqO=HryM>rfv*&J~xb;Io8ht|&%PxkH@uSexo781PhlYO=3y5a_oA z267U!v0(Xs$0IJk6Wh%3zV6~0TYTiDyd^j1*@JE=E1~hq_K<5f_Dx3l{f}25o$YrG zG={NkD6nr^+zHji3dM$qgvOpH`y_Ct$5y)3ICoT|m3ALph;rk?jcVP;omRVdcqrOJ?Ip1>B0o|^J*i%4CJd0Y;kf!zRH=EvVDsb9=)*j5<{YM;mjTmJ zL7H>&)zCT8F&#|L=Z;(t7P9#;up<-kE!!49qR%D~MSqN*qZbpa&hI&Y{cW~?UOOJs z9ex3jK#m3-TUo;mT^<+?hG&P))h>Ok>tdp9?A4dSG4oH*xK&<=rVC?%_1P%4hzF^i zs>L_loGyvi*Vd@`DV5#rOIpYzF7@ADKO>;=?5;+@_7<cgI|;|%VbiGF zAU}(88$a?#uu0AC7*@%X%-8fuljHd5mu*#^YcH9Ma+8oZP(}ujhfUpc5bzLELuJdU zS%~%ORe8F*d=4iBulK>cL*bh5-m|^@Y-MLO|EKt>TiBjH5InSH{BZV{(0b1mCdRk& z)7DcDr?s|iw-x0{_lnM~+MvaMy7ooN`=#xe9jiYB&4&i&dY&BrIr2JFjEpzR-wE0n zOcU2Qixrr{Y3i|8n!VMr3yFiBm7T2@2Vwi?8}$qINFqi>_s&&*MlrcZ>@><)_Pm-) z;^G(45iyd7;zZG6nKa#DSH^HwZo35VNvk#7$nhbeA`6utvtQvjf1ac5a*SxZDGzqr zy)`vg=6%KiDd%?nH3Cn#C1(8@?_N*2Y`S4w4@8$|fKar}k*DvBS82KRJ}!fLv9=ilXr4`q)F3&B{>YdZv{N+;{Gfb}&!$2jjd; za=1t;ob`I$fxH|#D6Bs1xf5;>Y7r_g@oGonW)e}VmKpFB;|j0hp;$sP)N$mIfusHia?8=-3XUgp{tdYAh(-T(?EKU?^c$h3SJ)dNfNqk6%}%QfTe>m^XQA`xQ(djL)D zZCuV_B?bddz34}dUqszbaZ(h>HqPP_zs*mrEK75JQ z=fM}A^g7r6;H&j2FM7p385R*`z3x=GcQeY=_vZT6nU~rNH-a1Ha8>GzM$DXW?Ro^# zjT-T*^dI3m|6s{U+jV6Sq8t27Zs(_qI$%jyWoJU*;V=DqQ?sbzOarBW`?KIByyJ@9qx z91)Bh`X@Kd6!||WMdm(vD>}a9`BAzvccz7)W5z5lEx72LsPeY0V~J~Gl2XcyGTcAu zYJP_+gWsF0iMMFfGtkp^GFM&`k%!eOqABhNt>~z9@}3t|TXM!4e0bwxO47K&2eaqa zO2uhmYWa7`Y@5@QU-YfSTC`c@Mx1Y)$>8oaDd_w9#IPXNVrp>siO~qDG^%j<6=5iC zQ&8+2CO@}|RwB*$#_fv^w7GAryak*z9PhbEsHt($sM8p>KW%qFm2@-~i+cUw;#=ZI zhcH6XxaQUig&&%G=cUYVVkUQb&#^sWn@LufmD7w0{((6?s8hU9z7Vwtvyv`YVEUlt zQ31YZmbTQaD7DA-@hHkzwXt}rs!7`Abk*{_)JOy?Dx$To zjl9w+%IqJCeVn}92&OVMYkM9OJ7-jL1>3?tmaVpjbdqhIpQj4LzC|%;(U0K=8>Uve zd436xW+zJp&-?Z!a*`RJ*QPr!U$GBK(XeyWx3gNz|99RCk<~6~1Y4j<}Av(REp|MOb8tZ|Vci6*fYq9TfYVEWd9$HePQ; zZ{d>FD$TxQ;$!ayHXTuWZEM5V+myT4)OECI#zea&@6}b4;>3^A=EceTvDCH{-R$x0 zYb0944;mY+t~mejE8+ey*_!%jgx(0d({B>;CNp@3o(7e^PAXH0KE_NxJ0rd1^|emi z>F+gM{1xeH5{WE9$zpdulr|=>5w1$-Yeup;a<|_Kbrt<$a6_3i9MAC?(6;V{UQbNK z2HB0vx-{DlpSZiJSj6mn`}TZa_u3kBRD8L3T6)_!gPJNwacYCBSEil2#V3FLcha2? z72}d4rlV9cB%@1SUDur#-r|jYW3Je#A+MMrPmYuFrQ{l6@|P{33-d&8bQnJc*PDV= zQxRFx?+P%5Yr28$Ptq<~n7wUnGlpJO=!ln$MTgnBy%5l|R8clF4oT$hZN8k;e8&z} zWE9vB*0pnlo_k-Ys8}d_x6w?py7~H8fo-QH-D?cmaXF?j(r{;4ak@I?LfPDpGN$d1 zMH9oH3$3NxM}Nj(){*R`{+uvDz>`sdK+zt?2(kvq+IJkvDAYO+qKh^v86!`7gBKEZ*`tT^dOax* zzE7&|iN|j99lRKT-`p--+bWEm92&P-YJDh}3~4NcGx`Sm?}qHb-iUZB+VrQ?JQ2A! zEuKNvrH6}Q*aN^+vs@}3;Fmn8j=3oj;7dTVX_|YJiagPH8lqw!zq)uC7zh?-4xYTf zk|pAY!8h_S0S8SXAq)9^-0I?0lFjRrB5HDN2LcDuG_3ocaFU(-CK!`@G%IM4pHmv2 zSETLCuTGM6ls(jYt_&bRgTMczD!yY!mvKAxz@W-DL5q1MCkGTC1s|RWzQp_J%Ub~t zaYBdh_xMwHtJTb2aeb4t={L5+^fofye7AsWp3O`_GE7gjJ}q-cRqF=cf&Nx&z$F^i z^K~9+rte8up9xkS;HwSU!(L|%7qtdFct9vnMw9QUGJB26Y7N+3Zy{&0g2LLnE?o2N zhoO(%0R*_YqaLdEhN%grWBXmgMIeB7QHVJ0od7h)-HX3f>{_C53A#ebd#VolZ`A7% zuK5?SOz>%o8sefR85%+q@WwC|sIIJ1T_L8rqD}?ZY*nDWg8Jgsi*PwQq9g^2S8~~k z5;Zqy!?i8nDI_W;DKtgOW#5swht@4tP6_cRl9h0k6j!08^7>5cVtaSxiMW;+x|r6O zgjPUESrna%ZRA+Zg@(o~RF($!2LJHTYwA?Y(wKqNF6s*n-4vmP<4$+L&CyVr636cg zuZX@6IH+<-+z<9|Tk(HuB>dJ01@p~_XVSwyUS5fTXqUiPHBy&(sv8YY`9skzV^QLw z58%*{qYvMq#V>hUh6oG4Z|WKd{OxIv1nvkbKRN-w81CJ~#idr$bKKewhR!A} zesg~P4F$AzhW`wW2l2BCsUf5Jwfu;x=Ps5{Fve@6gp2}~lfMua(lFoEw&Zqu_*&{~ z=pNtO_=q+Zl5IIx$%)58A}jfqn{h;@N}(OH!&RYh2Qr6}V2Hig=AFdd*KHnzlADPU z3)SaJa78=X_Em&7--ayI4LAtic(&!^(9_Ty`3OZ-=uDb6|2cxZi#oVI^bA(9|Wcg_LLqk@X#@g-u9 z8?Tv#5?h56nL&{PXej%2$rqg;hZjvL;h|0`jkRUmN>*p_wyIGmx#Iji<91sHd%2mnDXl$d83uKB@7S9Gfx74}EFGU<@AzVrF3K*Y*w;MHl>?Px9BxQJ&Um-0QeR zQ+{=SI{({LKA~jR^9Nbu%(3Cy{hsY@h1Q_E-R6n@I~RxwU)CCnjXrAIx^0?H6G$&M z8u%{TmtE@)k?_0kGDPy?Z$*g`sD*{TiRE=+l{}9LW?ujWLdXNL-?8|!8->d2Un7=T>^TT3v6cMR6zU1)YgHY2XL>8yR7JcUq)RVVZ54BY+LcHO_AdE;V3-`Y z)KEtuSKlHhyR~P-trBedX>k9&!XLK>E?h5mAU2x@&n3e^RpNz-QC8IoCD`*0#M0j^ zbsUHTxE{3Mv56F-k-E<$b)QA5P9Y{JoSl@$geO3chsK2H)l7Sd(_W=1MNK z81%hVtz`pKq<)`KRR&GsQbSe<9RaEsd5#$MIRkVe5eR6{NE&N6MFu3aqP7=y4u8a= z;VX^>x$WNjX0Kehz8CI3z62O8Xg2|;EK$^;?b4$GvP2mRoVnF3jIvk6EMXC!*i`ZF zs-06;(CwQbyR_AaqwA`#^eo*PBYoJwe)z&c{U=4w#P`8;nI^vVBK8+*jiqn`RBU)xUixRM%w|_dG^La`{3QDd!8&Nw-vc!xw!5cnf4~{KIlwl zNxqZJCtpC@eEIe|Yf{vuCaYY%AU0UDQf{vWd!vz=H#hlbo;qRgj_|!!I_HggTy*{;#Xh4 zz-}}jI>+{&#t%@8VzzW#jOn!k^gD`<&u%lRg?~;6F7U$ZPUDs1n$MjHvLO^`<*2F9 z+q6;s;k&Zv)ko-EVOfE-rM2S`PLQ@PxQ;pN602MOIedqC8>Itm_aZj^mi5g9+e?{N zVcuc#S4N99h_r6tD2vup>Y|0aD?J?aW|02eX*r?rz8@!zo5QS7z${W++P3r}V)QB& z>vPTI8j3lC&+{?FtaM+YY^=`$mG3?MXu6a(37e}2fVVVV2FHue;emPi0{cbDm>%rG zU#v_hRNf|USi&L@6u(LxR>9E_7@a=NqIy#_w7w8b0zeK5vIhqRt=Mp_B^2zm$@*Fl zU5yubF6f?#t<=$df!^%@>Fd0snp)a5uGla%sUkHvJSC?$!EgS&9?YX?w_UZb3DuJtDcbdfe(bx98mS%2m2AUsG;ka6eutpR}PY z8cE5|e56X?Hd6e$(pMi0*_zuvNM#ZKQ-~@ievUKqi*-tk=IQIyFO4kg&gcbeVND+g z=mzN4>&6(ht@*M#icH-@qJ*tGouC;EqC>Vg<0uvhlqH3I8)<#PUR_#MpS5~R z3p%*Ea!rWo$@>?t)??aKN+(0qO6?SU0)lJ1Op82U?5A%Q%{xn3Yev1$iaro9{i@zz zlp03efVtZvnwdQlGNLT^!02(p%I(X$n*zn9scr6*dSMu^=F18j* zF&{8bOZlfj@!P7;=_2~)C*!m5^_HS69qyZ*=hSHC)joU-#h#>WBO2XR_GMD$m`#r7 zJM87}%lOiLC9ssfbLVo#r=|hw9kwUq{e@S|Yi0wUC~(9rXmY)OeSC*;QKO>W$SKGD zX;vkyw{CE(H;x{S8Jd9@1ZfhW-U{emU_E?|T5;B7n#Uk^M1I(0$!a=9A!2@_`wqyANw&=wfuuTMr1bPPW0FhJ zuxw;07iy0K@yuDQMT~w)#HKw%-&oY}jmr0UF?w?2vukWgP_adz20{KA?V0?AyI8|E zQBL};t4}Ayv(bGY8hsHQn1 zs3xLM?542|Myc`C_|dPiCE|kWndatk84kJ|q3au`Q=Vfct#CmlL2a2N!!ae2k}AXG z^?ZDaZ<4jOxxW3_pHiG|+%Qz%dPf)E3hy&M6+TD7@Kf2ne@1aaoYx{x-7PEca;l0y+zme< z)qmmi+Vq8;0BSB{usg0HXQksIaQpPn_=ypR-uKMTt*^; zf=?4pBY`V+h~6!EcD_~pLWqw#UXy?lYreysHS=+&Viz9a@lq+Tj(^qUr6xfzH(v0m ze%@;_Tgz2mZ-rwoPjVfzj>`YM2`)IB31e28_5aTKwbNHnnE znX$%r<$Qs@5;d#Qp+u7jH`V3 z6db?Os4?<2Hj4uj?v8QvUcNE(I7YeKs%g@)+97C?#+GJt`00Gccx~;{1v^yFsDDG9 z0tYQ$%CiTI;rlmTiN8 zC%*pKBak)aswa~LJxhdP_et5f6A^mp^0zjf%0q+!X_uNW3(q-GTTNc~Y1R&h3d3H=%tY81i6z^I{_ zWH=oi>TujJc73Yq`nO|iHb-mAv5oR$*UYCr)u4vHvalJ3OJtg;TX4QH4~-+g7I%bxz{H zVHwVHwJ`)-OXO;D4is6`7No7CthPQm;{Ca!QF1vdrTf6XLNHeZLLVDg3jA0W<>c)c@=+I!$b&XdH3@I$@%aq=Y6HL+J{CT zPM?u)^PFtxdBfjgfEv&%Sdcb36syi$;M?E+#=?t`!(ipY6{!2i&)tiUkGGv{IeuVL z5b!EaQnBJaD>C+ZwN7+e{&v@P&Pdy{keBSA5|_)e@>$Bi2w%x0ip3>4yMQlvG^UKB zBjv8aMdty%^kM*Kz!_;?uDSFQyIb=nPsmoVKlV zeJu zt?hTWDzf6``z+iiGfZ$#PoTe6)>iQWl-qywV%l+JefegQr$LB-VDL~7|DIgNcKXNs zm63<_+QP>7it9hUmoTl}jah!W_gxJkEi{HJUkR#|^0ntI_ev(BH_#?|#-`5VAi4#^ z?lRN5_U~cLpF&@IzX2^TPO1q{T-p^wMT4ejhVg2t1?KJp`_c33qM{<9LK+_U@Ngxb z?{>){@27F!qfVAi;&pr$?Ly{KA`VHt`o2L&x5pz^my0I2KPWxgBXu5@ezy-9YqaB0 zrVANa^$fkXEwgAMOVzb0A~wAl*DZX{pjahYrM!O6f1@UgUCft*lVbU5ygQC@;hg*V zg>O2S_r7-#dUB`Jy;7CC7NtHVtmTeXe1bQ!>2*IEH(JLzPgTzGMKcx@<@=-*J;;k7 z&JEwDB3=qe2>DzQ=l_x6T~f@$|Z(MJ)6(&0ndV~67AFcn*M~2Np^p%6g$gprL4U(K2Vyc z?{GL6Sg{mW_ua(VtzmMD7EFVBcVqkf?MY zbZU_4Bg+O|4Ly_PgJmbu2TS={lN-Q|?qCMDC>G#{Lr>LabZT09WpueGi@=^-T3Wh6 zVZg3sY5>L+=~iR7$i#8;wMxF8?FLQv)=*5uTO%9M5Q}(?3A>68{l|mhvjP^%H1hJa z?D7UN)Mq=^>uAoB-haVG4{ze4{pr#BU3fK%uwv~YNRu_LB{%T;uD80-Tj{25>Ms(h z-0#w)l?qw)_d(BLNK5p9P}QX#hoypT63!?Lyy@e)J)W9y*dxds^zNq3Vp>W-Ms3!3 zx#vVGdzQU=)$VRvV(HA;0pCCPw4kyL@+)6>UL_9+9NAUVO z*cRaF=#vGaio!tZF{YpZ5dDfZax_-d+V z)GzLVbXIGsQWE&Kn2qoEzP)YlAn^!WbswZGAAd`Q`3=G95l?gfMNSdnW{BRGZAKYQE^xqYJJ5U@ZsbO&7QO@#p049Cjkz zAjf1$fpbONdmP_8T?$K%9gL!=nPezW5Kz>YmE;q7gdC1Eikgd9I}hWK6!DO|2aL+C z4qJxQ9p|?efBfT-OHktgqs>Lf(98#prQ z5_D`5jL{6r=Pl!4XojqZdHHzv1P4LCie@Nlmw|)m=ybl1Auw z-0DJdx&jouSG2Ojqb_9kDyb=oyK&FxUO{p@mNi>F22N6y4js@lFG%V({C?tJ39;F+ zD3VNNdk~sqv+$DG)nb{&qiC$gYfw?X^N=GCqBbWcZ-`)qa9wByNR1SF=F4b8eVfTx|2uFvOe z@vY9}Y*{&}bW47e$E(?vbdC!@0TeA0BrsRI1VL5d_6>sSKJZ<8W=-gu%gQv}4zv6% zWug-LSn|vW=@{4f9l`N;w9{uXPG?r#_G37k?N1&uH@Td_(lY_dq_1ObbNT9xErE-n>2Kns_9NfKiBS}`JCXDi?sSZ@@30@@ zC!42E`Lr>TXr9l$Bhl=i_yg%XZMC2ACeoYGx#c7u^pg8sQO=vB`BQIiv24lN;RRmO zNfc!)x6&$fZOyoNx$a>Z4XeWm(UfDc7Vq1)kaf+9gqAI&L;-SdRphYUuGMpg{RZ>i zYRHy|U5ELW$W}@IavM9v6u#C=VzNc*<#}Hx`LXyD6jQiBF9}-W^_S;kS1HEsv{J~g zGK?ki5^(5QAv@Q`waR8ELKk{gk>uF8reud~TsyvnweYixhHS5D&SdDbA$VWg+|Lhd1jEC5AS`m4`M9J$4PZ7lAiN*s`O>y&u2E9@nGMx&33LiT`33m~{KbkIAiV)IQPm&ECDi~F(o5eIxZ{O% z3j_l&mTI6A*$h+`f)IyEOTvK&xdU=Y3(F09uvY0)UKNVBjD4*uOAT*l%1Vx$a*r{`58lvN8 zN+`wH2sfS#>*X~|34OTFv5~&+@h2K})zvlHXM)EHa{-Y^!5D@ FromConfig: pass -class MockPDFPreprocessor(PDFPreprocessor): - def __init__(self, res: list[list[Path] | Exception]): - self._res = iter(res) - - def __call__( - self, - pdf_path: Path, # noqa: ARG002 - pdf_bytes: bytes, # noqa: ARG002 - output_dir: Path, # noqa: ARG002 - ) -> list[Path]: - r = next(self._res) - if isinstance(r, Exception): - raise r - return r - - @pytest.fixture def symlinked_doc_0() -> DatashareFile: # Let's test with a symlinked file diff --git a/workers/passport-worker/tests/test_liteparse.py b/workers/passport-worker/tests/test_liteparse.py new file mode 100644 index 00000000..37d1bc8f --- /dev/null +++ b/workers/passport-worker/tests/test_liteparse.py @@ -0,0 +1,99 @@ +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from liteparse.types import LayoutComplexityStats, PageComplexityStats + +from passport_worker.liteparse_ import ( + LayoutReason, + LiteparsePDFPreprocessor, + LiteparsePDFPreprocessorConfig, + OCRReason, + may_have_passport, +) +from passport_worker.preprocessing import PDFPreprocessor +from tests import TEST_RESOURCE_PATH + + +def test_liteparse_pdf_preprocessor_from_config() -> None: + # Given + config = LiteparsePDFPreprocessorConfig() + # When + processor = PDFPreprocessor.from_config(config) + # Then + assert isinstance(processor, LiteparsePDFPreprocessor) + + +async def test_should_skip_pages(tmp_path: Path) -> None: + # Given + path = TEST_RESOURCE_PATH / "liteparse_test.pdf" + output_dir = Path(tmp_path) + inner = AsyncMock() + pages = [Path("some_pdf_page_as.png")] + inner.process_pdf.return_value = pages + config = LiteparsePDFPreprocessorConfig() + preprocessor = LiteparsePDFPreprocessor(config, inner) + + # When + res = await preprocessor(path, output_dir=output_dir, force_reprocessing=True) + + # Then + assert res == pages + expected_skip = {0} + inner.process_pdf.assert_called_once_with( + path, expected_skip, output_dir=output_dir, force_reprocessing=True + ) + + +@pytest.fixture(scope="session") +def test_complexity() -> PageComplexityStats: + return PageComplexityStats( + page_number=0, + text_length=2, + text_coverage=0.5, + has_substantial_images=True, + image_block_count=1, + image_coverage=0.5, + largest_image_coverage=0.8, + full_page_image=True, + uncovered_vector_area=0.5, + is_garbled=True, + page_area=0.5, + needs_ocr=True, + reasons=["scanned"], + layout=LayoutComplexityStats( + column_count=1, + ruled_table_count=1, + ruled_table_coverage=0.5, + text_table_run_count=1, + figure_count=1, + figure_coverage=0.5, + is_complex=True, + reasons=["dense-graphics"], + ), + ) + + +@pytest.mark.parametrize( + ("ocr_reasons", "layout_reasons", "expected"), + [ + (set(), set(), False), + ({OCRReason.SCANNED}, set(), True), + ({OCRReason.EMBEDDED_IMAGES}, set(), False), + (set(), {LayoutReason.DENSE_GRAPHICS}, True), + (set(), {LayoutReason.TABLE_LIKELY}, False), + ], +) +async def test_may_have_passport( + test_complexity: PageComplexityStats, + ocr_reasons: set[OCRReason], + layout_reasons: set[LayoutReason], + *, + expected: bool, +) -> None: + # When + might_have_passport = may_have_passport( + test_complexity, ocr_reasons=ocr_reasons, layout_reasons=layout_reasons + ) + # Then + assert might_have_passport is expected diff --git a/workers/passport-worker/tests/test_preprocessing.py b/workers/passport-worker/tests/test_preprocessing.py index e56d6b0f..82c6378b 100644 --- a/workers/passport-worker/tests/test_preprocessing.py +++ b/workers/passport-worker/tests/test_preprocessing.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Self import pytest from datashare_python.conftest import TEST_PROJECT @@ -83,10 +84,9 @@ def __init__(self, res: list[list[Path] | Exception]): self._res = iter(res) self.processed = [] - def __call__( + async def __call__( self, pdf_path: Path, - pdf_bytes: bytes, # noqa: ARG002 output_dir: Path, # noqa: ARG002 *, force_reprocessing: bool, @@ -99,6 +99,9 @@ def __call__( self.processed.append(pdf_path) return r + @classmethod + def _from_config(cls, config: RegistrableConfig, **extras) -> Self: ... + @pytest.fixture def symlinked_doc_0() -> DatashareFile: @@ -289,14 +292,14 @@ async def test_convert_to_pdfs_act_caching( @pytest.fixture def doc_1_pages( - test_worker_config: PassportWorkerConfig, doc_0: Document + test_worker_config: PassportWorkerConfig, doc_1: Document ) -> list[Path]: worker_paths = test_worker_config.paths workdir = worker_paths.workdir output_root = workdir.joinpath(TEST_PROJECT, "workflow_id") output_root.mkdir(parents=True, exist_ok=True) - doc_0_page_dir = output_root / safe_dir(doc_0.id) / doc_0.id - doc_0_pages = [doc_0_page_dir / "page_0.png", doc_0_page_dir / "page_1.png"] + doc_1_page_dir = output_root / safe_dir(doc_1.id) / doc_1.id + doc_0_pages = [doc_1_page_dir / "page_0.png", doc_1_page_dir / "page_1.png"] return doc_0_pages @@ -318,9 +321,9 @@ async def test_preprocess_pdfs_act( batch_path.write_text("\n".join(d.model_dump_json() for d in batch)) # When successes, errors = await preprocess_pdfs_act( + preprocessor, batch_path, worker_paths, - preprocessor, output_root=output_root, force_reprocessing=True, ) @@ -362,9 +365,9 @@ async def test_preprocess_pdfs_act_caching( p.write_bytes((DOCS_PATH / "passport.png").read_bytes()) # When successes, errors = await preprocess_pdfs_act( + preprocessor, batch_path, worker_paths, - preprocessor, output_root=output_root, force_reprocessing=False, ) diff --git a/workers/passport-worker/uv.dist.lock b/workers/passport-worker/uv.dist.lock index d7f2cbff..6409a98c 100644 --- a/workers/passport-worker/uv.dist.lock +++ b/workers/passport-worker/uv.dist.lock @@ -297,6 +297,7 @@ dependencies = [ { name = "datashare-python" }, { name = "icij-common", extra = ["elasticsearch-8"] }, { name = "icij-passport-core" }, + { name = "liteparse" }, ] [package.optional-dependencies] @@ -324,10 +325,11 @@ dev = [ requires-dist = [ { name = "datashare-python", specifier = "~=0.12.12" }, { name = "icij-common", extras = ["elasticsearch-8"], specifier = "~=0.9.3" }, - { name = "icij-passport-core", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.9" }, + { name = "icij-passport-core", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.13" }, + { name = "liteparse", specifier = ">=2.19.0" }, { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -537,15 +539,15 @@ elasticsearch-8 = [ [[package]] name = "icij-passport-core" -version = "0.11.9" +version = "0.11.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiostream" }, { name = "icij-common" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/b3/034a1e061d3b9bf9df8791b476fb4859566180e574db36c8b7c90fb2c85f/icij_passport_core-0.11.9.tar.gz", hash = "sha256:56c519c8c2a4d2193f5a240a999c3e3f7ddbcfed3a82e388e2b4aba9f7acadcc", size = 14490527, upload-time = "2026-11-10T17:40:00.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2e/6cf9adb636492021cc1c58e0df00cb5941afa7bbc06921b2cb7133efe27d/icij_passport_core-0.11.13.tar.gz", hash = "sha256:6e6ba913ec7ed887645b20c16be705fa9343c420b38949c097ae89d5408fe367", size = 14490699, upload-time = "2026-11-18T16:11:21.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/45/4e6e4c63fdb37b8744077b4f3b54940f0659429506d08d73378c80f4c43e/icij_passport_core-0.11.9-py3-none-any.whl", hash = "sha256:4a859da648994c731771c3538100e64c48404616e2fd7c4be7e1cf45523203af", size = 10959056, upload-time = "2026-11-10T17:39:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ff/410d9f224bfe7f2c3db953d848e73559dfc4df8e61eb84df577a9555dbca/icij_passport_core-0.11.13-py3-none-any.whl", hash = "sha256:a77d693b8bf21067e5f7162c3a2188eb0a5920a6d4af04ec2ad1fd86f04c8efe", size = 10959255, upload-time = "2026-11-18T16:11:18.964Z" }, ] [package.optional-dependencies] @@ -715,6 +717,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/04/e8d3b4e89cba70babf4bf58fce963dc6f627ae8192d2a45b495f42422880/Levenshtein-0.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:5a61606bad3afb9fcec0a2a21871319c3f7da933658d2e0e6e55ab4a34814f48", size = 90919, upload-time = "2023-10-09T13:38:52.507Z" }, ] +[[package]] +name = "liteparse" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/c1/562179f34bc184cb874d7f4fefb23c6f471d007c9cb5e5a1df47d4a73c3e/liteparse-2.19.0.tar.gz", hash = "sha256:b90f6cb350611853f6142edafefb715b5973ca8a2998e36b8837781332a089de", size = 688596, upload-time = "2026-11-13T02:37:49.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/15/403d15c29c40c9a20352d632ff0c679b907f0e7c17294607a313b0d8099f/liteparse-2.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffbc26835590160313fec3736c762ae57088d806adc4395dc87090e4e5607993", size = 11752816, upload-time = "2026-11-13T02:37:35.3Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/96128effcacc673e35a0313e34ca461199d08cdc3cb3e04807e461bd1c02/liteparse-2.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:52742af76cbf7dfb4b451d417eb99d87d1dacfceb2bfb89849db5e6b052f2618", size = 11203396, upload-time = "2026-11-13T02:37:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d0/ded0c9c41d71d90fe6ac9889d952ef44d89099d3b85301d8952d316abfcc/liteparse-2.19.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7eea6b66ff5b718f4672c45032ba642e836aef005f782836c73b261f2991a57f", size = 12692857, upload-time = "2026-11-13T02:37:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/fe/43/492849721200c35822e5f6eb6517170bd306bc5cfe96ba1bbf910b67e837/liteparse-2.19.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8d1663f742fd4b8e504ebea757a219658b10755088a619243858e376f9f98531", size = 12956108, upload-time = "2026-11-13T02:37:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/d96ec3444733d03a22a8cad1b592bce0501feef8f26e5e2a0ed88bc50f71/liteparse-2.19.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1c09f7911d502e044c8f41807213534808ffbd40a64d8abce053ef0699c6d5a1", size = 16089685, upload-time = "2026-11-13T02:37:43.256Z" }, + { url = "https://files.pythonhosted.org/packages/78/24/ff81674f6d338199e27ba5dd545925202e3bca8ea371acaa42818a652c7c/liteparse-2.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:f58b5e1eda4ac2fbcc17346c9e40162ed9e99c7a8bad95dff50f336e4bbb2f6f", size = 11146829, upload-time = "2026-11-13T02:37:45.566Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4f/0a200526b7858704ed6841c46702cc417bb27aa56fff579a6f5ab3a037ff/liteparse-2.19.0-cp310-abi3-win_arm64.whl", hash = "sha256:6daddd5770a56761ab4f0afd6b96d5a8670c845c9df06882a67eafcc21c05242", size = 10604750, upload-time = "2026-11-13T02:37:48.314Z" }, +] + [[package]] name = "lru-dict" version = "1.4.1" diff --git a/workers/passport-worker/uv.lock b/workers/passport-worker/uv.lock index f867ca00..361935dc 100644 --- a/workers/passport-worker/uv.lock +++ b/workers/passport-worker/uv.lock @@ -297,6 +297,7 @@ dependencies = [ { name = "datashare-python" }, { name = "icij-common", extra = ["elasticsearch-8"] }, { name = "icij-passport-core" }, + { name = "liteparse" }, ] [package.optional-dependencies] @@ -324,10 +325,11 @@ dev = [ requires-dist = [ { name = "datashare-python", editable = "../../datashare-python" }, { name = "icij-common", extras = ["elasticsearch-8"], specifier = "~=0.9.3" }, - { name = "icij-passport-core", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.9" }, - { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.9" }, + { name = "icij-passport-core", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["cpu", "inference"], marker = "extra == 'inference-cpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["gpu", "inference"], marker = "extra == 'inference-gpu'", specifier = "==0.11.13" }, + { name = "icij-passport-core", extras = ["preprocessing"], marker = "extra == 'preprocessing'", specifier = "==0.11.13" }, + { name = "liteparse", specifier = ">=2.19.0" }, { name = "pandas", marker = "extra == 'preprocessing'", specifier = "~=3.0" }, ] provides-extras = ["inference-cpu", "inference-gpu", "preprocessing"] @@ -561,15 +563,15 @@ elasticsearch-8 = [ [[package]] name = "icij-passport-core" -version = "0.11.9" +version = "0.11.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiostream" }, { name = "icij-common" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/b3/034a1e061d3b9bf9df8791b476fb4859566180e574db36c8b7c90fb2c85f/icij_passport_core-0.11.9.tar.gz", hash = "sha256:56c519c8c2a4d2193f5a240a999c3e3f7ddbcfed3a82e388e2b4aba9f7acadcc", size = 14490527, upload-time = "2026-11-10T17:40:00.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2e/6cf9adb636492021cc1c58e0df00cb5941afa7bbc06921b2cb7133efe27d/icij_passport_core-0.11.13.tar.gz", hash = "sha256:6e6ba913ec7ed887645b20c16be705fa9343c420b38949c097ae89d5408fe367", size = 14490699, upload-time = "2026-11-18T16:11:21.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/45/4e6e4c63fdb37b8744077b4f3b54940f0659429506d08d73378c80f4c43e/icij_passport_core-0.11.9-py3-none-any.whl", hash = "sha256:4a859da648994c731771c3538100e64c48404616e2fd7c4be7e1cf45523203af", size = 10959056, upload-time = "2026-11-10T17:39:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ff/410d9f224bfe7f2c3db953d848e73559dfc4df8e61eb84df577a9555dbca/icij_passport_core-0.11.13-py3-none-any.whl", hash = "sha256:a77d693b8bf21067e5f7162c3a2188eb0a5920a6d4af04ec2ad1fd86f04c8efe", size = 10959255, upload-time = "2026-11-18T16:11:18.964Z" }, ] [package.optional-dependencies] @@ -739,6 +741,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/04/e8d3b4e89cba70babf4bf58fce963dc6f627ae8192d2a45b495f42422880/Levenshtein-0.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:5a61606bad3afb9fcec0a2a21871319c3f7da933658d2e0e6e55ab4a34814f48", size = 90919, upload-time = "2023-10-09T13:38:52.507Z" }, ] +[[package]] +name = "liteparse" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/c1/562179f34bc184cb874d7f4fefb23c6f471d007c9cb5e5a1df47d4a73c3e/liteparse-2.19.0.tar.gz", hash = "sha256:b90f6cb350611853f6142edafefb715b5973ca8a2998e36b8837781332a089de", size = 688596, upload-time = "2026-11-13T02:37:49.949Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/15/403d15c29c40c9a20352d632ff0c679b907f0e7c17294607a313b0d8099f/liteparse-2.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffbc26835590160313fec3736c762ae57088d806adc4395dc87090e4e5607993", size = 11752816, upload-time = "2026-11-13T02:37:35.3Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/96128effcacc673e35a0313e34ca461199d08cdc3cb3e04807e461bd1c02/liteparse-2.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:52742af76cbf7dfb4b451d417eb99d87d1dacfceb2bfb89849db5e6b052f2618", size = 11203396, upload-time = "2026-11-13T02:37:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d0/ded0c9c41d71d90fe6ac9889d952ef44d89099d3b85301d8952d316abfcc/liteparse-2.19.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7eea6b66ff5b718f4672c45032ba642e836aef005f782836c73b261f2991a57f", size = 12692857, upload-time = "2026-11-13T02:37:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/fe/43/492849721200c35822e5f6eb6517170bd306bc5cfe96ba1bbf910b67e837/liteparse-2.19.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8d1663f742fd4b8e504ebea757a219658b10755088a619243858e376f9f98531", size = 12956108, upload-time = "2026-11-13T02:37:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/f7/c3/d96ec3444733d03a22a8cad1b592bce0501feef8f26e5e2a0ed88bc50f71/liteparse-2.19.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1c09f7911d502e044c8f41807213534808ffbd40a64d8abce053ef0699c6d5a1", size = 16089685, upload-time = "2026-11-13T02:37:43.256Z" }, + { url = "https://files.pythonhosted.org/packages/78/24/ff81674f6d338199e27ba5dd545925202e3bca8ea371acaa42818a652c7c/liteparse-2.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:f58b5e1eda4ac2fbcc17346c9e40162ed9e99c7a8bad95dff50f336e4bbb2f6f", size = 11146829, upload-time = "2026-11-13T02:37:45.566Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4f/0a200526b7858704ed6841c46702cc417bb27aa56fff579a6f5ab3a037ff/liteparse-2.19.0-cp310-abi3-win_arm64.whl", hash = "sha256:6daddd5770a56761ab4f0afd6b96d5a8670c845c9df06882a67eafcc21c05242", size = 10604750, upload-time = "2026-11-13T02:37:48.314Z" }, +] + [[package]] name = "lru-dict" version = "1.4.1"