Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datashare-python/datashare_python/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1111,7 +1111,7 @@ async def run_with_concurrency(
) -> AsyncGenerator:
max_concurrency = asyncio.Semaphore(max_concurrency)
aws = (_run_with_semaphore(aw, max_concurrency) for aw in aws)
async for res in asyncio.as_completed(aws):
for res in asyncio.as_completed(aws):
yield await res


Expand Down
14 changes: 12 additions & 2 deletions workers/passport-worker/passport_worker/activities.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@
PassportDetectionResponse,
PassportExportArgs,
PassportExportResponse,
PDFPreprocessorConfig,
PreprocessingBatches,
PreprocessingConfig,
)
from .preprocessing import (
ImagePreprocessor,
PDFConverter,
PDFPreprocessor,
convert_to_pdfs_act,
preprocess_images_act,
preprocess_pdfs_act,
Expand Down Expand Up @@ -177,7 +179,7 @@ async def convert_to_pdfs(
] = None,
) -> tuple[Path, Path]:
worker_config = cast(PassportWorkerConfig, lifespan_worker_config())
config = worker_config.preprocessing.pdfs
config = worker_config.preprocessing.pdfs.conversion
cache = lifespan_pdf_converter_cache()
cache_key = config_cache_key(config.pdf_converter)
pdf_converter_factory = async_enter_cm(
Expand Down Expand Up @@ -215,19 +217,27 @@ async def preprocess_pdfs(
self,
batch: Path,
project: str,
config: PDFPreprocessorConfig,
*,
force_reprocessing: bool,
progress: Annotated[
AsyncProgressRateHandler | None, Weight(value=_PREPROCESS_PDF_WEIGHT)
] = None,
) -> tuple[Path, Path]:
worker_config = cast(PassportWorkerConfig, lifespan_worker_config())
worker_pdf_config = worker_config.preprocessing.pdfs.preprocessing
workdir = worker_config.paths.workdir
logger.info("loading pdf preprocessor...")
# We don't cache the pdf preprocessor, it's lightweight
extras = worker_pdf_config.resolve_preprocessor_extras(config)
pdf_preprocessor = PDFPreprocessor.from_config(config, **extras)
# We cache processing at the config level, we ignore package updates,
# to discard the cache we just need to disable it in the args to overwrite
output_root = activity_workdir(workdir, project, caching_key="pdf-preprocessor")
result_cache_key = config_cache_key(config)
output_root = activity_workdir(workdir, project, caching_key=result_cache_key)
output_root.mkdir(parents=True, exist_ok=True)
successes, errors = await preprocess_pdfs_act(
pdf_preprocessor,
batch,
worker_config.paths,
output_root=output_root,
Expand Down
44 changes: 42 additions & 2 deletions workers/passport-worker/passport_worker/config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
from concurrent.futures import ProcessPoolExecutor
from datetime import timedelta
from enum import StrEnum
from typing import ClassVar
from typing import Any, ClassVar

import datashare_python
from datashare_python.config import (
Expand All @@ -15,6 +16,8 @@
from icij_common.registrable import RegistrableConfig
from pydantic import Field

from .objects import DefaultPDFPreprocessorConfig, PDFPreprocessorConfigBase

_ALL_LOGGERS = [datashare_python.__name__, __name__, "__main__"]

_DEFAULT_LOGGERS = {
Expand Down Expand Up @@ -68,13 +71,50 @@ class PDFConversionWorkerConfig(DatashareModel):
)


class LiteparsePDFPreprocessorWorkerConfig(BaseModel):
concurrency: int = Field(default_factory=os.cpu_count)
page_batch_size: int = 8


class PDFPreprocessingWorkerConfig(BaseModel):
liteparse: LiteparsePDFPreprocessorWorkerConfig = Field(
default_factory=LiteparsePDFPreprocessorWorkerConfig
)

def resolve_preprocessor_extras(
self, preprocessor_config: PDFPreprocessorConfigBase
) -> dict[str, Any]:
from .liteparse_ import LiteparsePDFPreprocessorConfig # noqa: PLC0415

match preprocessor_config:
case DefaultPDFPreprocessorConfig():
return dict()
case LiteparsePDFPreprocessorConfig():
return self.liteparse.model_dump()
raise TypeError(f"unexpected type: {preprocessor_config.__class__.__name__}")


class PDFProcessingWorkerConfig(BaseModel):
conversion: PDFConversionWorkerConfig = Field(
default_factory=PDFConversionWorkerConfig
)
preprocessing: PDFPreprocessingWorkerConfig = Field(
default_factory=PDFPreprocessingWorkerConfig
)

def resolve_pdf_preprocessor_extras(
self, preprocessor_config: PDFPreprocessorConfigBase
) -> dict[str, Any]:
return self.preprocessing.resolve_preprocessor_extras(preprocessor_config)


class PreprocessingWorkerConfig(DatashareModel):
target_n_pages_per_batch: int = 200

images: ImagePreprocessingWorkerConfig = Field(
default_factory=ImagePreprocessingWorkerConfig
)
pdfs: PDFConversionWorkerConfig = Field(default_factory=PDFConversionWorkerConfig)
pdfs: PDFProcessingWorkerConfig = Field(default_factory=PDFProcessingWorkerConfig)

def to_image_preprocessing_executor(self) -> ProcessPoolExecutor:
return self.images.to_image_preprocessing_executor()
Expand Down
110 changes: 110 additions & 0 deletions workers/passport-worker/passport_worker/liteparse_.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import asyncio
import logging
import os
from collections.abc import Iterator
from pathlib import Path
from typing import Self

from datashare_python.utils import run_with_concurrency
from liteparse.parser import LiteParse
from liteparse.types import PageComplexityStats

from .objects import (
LayoutReason,
LiteparsePDFPreprocessorConfig,
OCRReason,
PDFPreprocessorType,
)
from .preprocessing import DefaultPDFPreprocessor, PDFPreprocessor

logger = logging.getLogger(__name__)


@PDFPreprocessor.register(PDFPreprocessorType.LITEPARSE)
class LiteparsePDFPreprocessor(PDFPreprocessor[LiteparsePDFPreprocessorConfig]):
def __init__(
self,
config: LiteparsePDFPreprocessorConfig,
inner: DefaultPDFPreprocessor,
page_batch_size: int = 8,
concurrency: int | None = None,
):
super().__init__(config)
self._inner = inner
self._page_batch_size = page_batch_size
self._concurrency = concurrency or os.cpu_count()

async def __call__(
self, path: Path, *, output_dir: Path, force_reprocessing: bool
) -> list[Path]:
complexities = await self._compute_complexities(path, force_reprocessing)
ocr_reasons = self._config.page_filter.accept_ocr_reasons
layout_reasons = self._config.page_filter.accept_layout_reasons
skip = set()
for i, c in enumerate(complexities):
might_be_passport = may_have_passport(
c, ocr_reasons=ocr_reasons, layout_reasons=layout_reasons
)
if might_be_passport:
skip.add(i)
pages = await self._inner.process_pdf(
path, skip, output_dir=output_dir, force_reprocessing=force_reprocessing
)
return pages

async def _compute_complexities(
self, path: Path, n_pages: int
) -> list["PageComplexityStats"]:
batches = (
pages for pages in _chunk_docs(n_pages, chunk_size=self._page_batch_size)
)
aws = (_compute_batch_complexities(path, batch) for batch in batches)
complexities = []
async for res in run_with_concurrency(aws, self._concurrency):
complexities.extend(res)
complexities = sorted(complexities, key=lambda c: c.page_number)
return complexities

@classmethod
def _from_config(cls, config: LiteparsePDFPreprocessorConfig, **extras) -> Self: # noqa: ARG003
inner = DefaultPDFPreprocessor(config.preprocessor)
return cls(config, inner)


def may_have_passport(
complexity: PageComplexityStats,
ocr_reasons: set[OCRReason],
layout_reasons: set[LayoutReason],
) -> bool:
if any(r in ocr_reasons for r in complexity.reasons):
return True
if complexity.layout is not None:
return any(r in layout_reasons for r in complexity.layout.reasons)
return False


Range = tuple[int, int]


def _chunk_docs(n_pages: int, *, chunk_size: int) -> Iterator[Range]:
for i in range(0, n_pages, chunk_size):
start = i
end = min(start + n_pages, n_pages)
rng = (start, end)
yield rng


async def _compute_batch_complexities(
path: Path, rng: Range
) -> list[PageComplexityStats]:
target_pages = _liteparse_range(rng)
parser = LiteParse(ocr_enabled=False, target_pages=target_pages)
with parser:
comps = await asyncio.to_thread(parser.is_complex, path)
return comps


def _liteparse_range(rng: Range) -> str:
# page range is a Python range, liteparse indexes from 1 and upper bound
# is inclusive
return f"{rng[0] + 1}-{rng[1]}"
65 changes: 63 additions & 2 deletions workers/passport-worker/passport_worker/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
WorkerFile,
WorkerPaths,
)
from icij_common.pydantic_utils import safe_copy
from icij_common.pydantic_utils import make_enum_discriminator, safe_copy, tagged_union
from icij_common.registrable import RegistrableConfig
from passport_service.constants import (
DEFAULT_DETECTION_THRESHOLD,
Expand All @@ -30,7 +30,7 @@
DEFAULT_NMS_THRESHOLD,
)
from passport_service.objects import Passport
from pydantic import Field
from pydantic import Discriminator, Field

DocumentSearchQuery = dict[str, Any]
DocId = str
Expand Down Expand Up @@ -71,12 +71,73 @@ class DefaultImagePreprocessorConfig(ImagePreprocessorConfigBase):
ImagePreprocessorConfig = DefaultImagePreprocessorConfig


class PDFPreprocessorType(StrEnum):
DEFAULT = "default"
LITEPARSE = "liteparse"


class PDFPreprocessorConfigBase(DatashareModel, RegistrableConfig):
registry_key: ClassVar[str] = Field(frozen=True, default="type")
type: ClassVar[PDFPreprocessorType]


class DefaultPDFPreprocessorConfig(PDFPreprocessorConfigBase):
type: ClassVar[PDFPreprocessorType] = Field(
frozen=True, default=PDFPreprocessorType.DEFAULT
)


class OCRReason(StrEnum):
SCANNED = "scanned"
NO_TEXT = "no-text"
SPARSE_TEXT = "sparse-text"
EMBEDDED_IMAGES = "embedded-images"
GARBLED = "garbled"
VECTOR_TEXT = "vector-text"
ANNOTATION_TEXT = "annotation-text"


class LayoutReason(StrEnum):
MULTI_COLUMN = "multi-column"
TABLE_LIKELY = "table-likely"
DENSE_GRAPHICS = "dense-graphics"


def _all_ocr_reasons() -> set[OCRReason]:
return set(OCRReason)


class LiteparsePageFilter(BaseModel):
accept_ocr_reasons: set[OCRReason] = Field(default_factory=_all_ocr_reasons)
accept_layout_reasons: set[LayoutReason] = Field(default_factory=set)


class LiteparsePDFPreprocessorConfig(PDFPreprocessorConfigBase):
type: ClassVar[PDFPreprocessorType] = Field(
frozen=True, default=PDFPreprocessorType.LITEPARSE
)
preprocessor: DefaultPDFPreprocessorConfig = Field(
default_factory=DefaultPDFPreprocessorConfig
)
page_filter: LiteparsePageFilter = Field(default_factory=LiteparsePageFilter)


PDFPreprocessorConfig = tagged_union(
PDFPreprocessorConfigBase.__subclasses__(), lambda t: t.type.default
)
pdf_preprocessor_discriminator = make_enum_discriminator("type", PDFPreprocessorType)


class PreprocessingConfig(DatashareModel):
use_caching: bool = True

images: ImagePreprocessorConfig = Field(
default_factory=DefaultImagePreprocessorConfig
)
pdfs: PDFPreprocessorConfig = Field(
discriminator=Discriminator(pdf_preprocessor_discriminator),
default_factory=DefaultPDFPreprocessorConfig,
)


class PassportDetectorType(StrEnum):
Expand Down
Loading
Loading