Configuration API

scinr.newton.config.configure

configure(
    llm=None,
    repair_llm=None,
    neo4j_uri=None,
    neo4j_user=None,
    neo4j_password=None,
    enabled_base_themes=None,
    enabled_user_themes=None,
    extra_models_paths=None,
    storage_backend=None,
    mongodb_uri=None,
    mongodb_database=None,
    mongodb_raw_files_collection=None,
    mongodb_pages_collection=None,
    mongodb_gridfs_bucket=None,
    custom_storage=None,
    extra_converters=None,
    mistral_api_key=None,
    prompt_caching_enabled=None,
    extraction_batch_size=None,
    llm_concurrency=None,
    neo4j_concurrency=None,
    log_level="INFO",
    prompt_family=None,
    normalization_enabled=None,
    normalization_batch_size=None,
    normalization_llm=None,
)

Configure the scinr-ingest library.

Parameter resolution order: explicit argument > environment variable > default.

Parameters:

Name	Type	Description	Default
llm	Any | None	LangChain BaseChatModel instance to use for all LLM calls.	None
repair_llm	Any | None	LangChain BaseChatModel for the JSON repair loop. Falls back to llm if None.	None
neo4j_uri	str | None	Neo4j connection URI. Env: NEO4J_URI. Default: bolt://localhost:7687.	None
neo4j_user	str | None	Neo4j username. Env: NEO4J_USER.	None
neo4j_password	str | None	Neo4j password. Env: NEO4J_PASSWORD.	None
enabled_base_themes	list[ThemePath | str] | None	Whitelist of built-in theme paths to activate (ThemePath values).	None
enabled_user_themes	list[str] | None	Whitelist of user theme paths to activate.	None
extra_models_paths	list[str | Path] | None	Filesystem paths to scan for additional user-defined theme models.	None
storage_backend	Literal['none', 'mongodb', 'custom'] | None	Storage type: 'none' (default), 'mongodb', or 'custom'.	None
mongodb_uri	str | None	MongoDB connection URI. Env: MONGODB_URI.	None
mongodb_database	str | None	MongoDB database name. Env: MONGODB_DATABASE.	None
mongodb_raw_files_collection	str | None	Collection for raw file metadata.	None
mongodb_pages_collection	str | None	Collection for converted pages.	None
mongodb_gridfs_bucket	str | None	GridFS bucket name for binary files.	None
custom_storage	tuple | None	Tuple (RawFileRepository, PageRepository) when storage_backend='custom'.	None
extra_converters	dict[str, type] | None	Dict mapping file extensions to custom BaseConverter subclasses.	None
mistral_api_key	str | None	Mistral API key for PDF OCR conversion.	None
prompt_caching_enabled	bool | None	Enable prompt caching for supported LLM providers.	None
extraction_batch_size	int | None	Pages per extraction chunk (default: 1).	None
llm_concurrency	int | None	Max concurrent LLM calls (semaphore size, default: 4).	None
neo4j_concurrency	int | None	Max concurrent Neo4j write sessions (default: 10).	None
log_level	str	Logging level string ("DEBUG", "INFO", "WARNING", "ERROR").	'INFO'
prompt_family	PromptFamily | Literal['generic', 'claude', 'gpt_reasoning'] | None	Prompt family to use ("generic", "claude", or "gpt_reasoning").	None
normalization_enabled	bool | None	Enable post-extraction normalization for tabular data.	None
normalization_batch_size	int | None	Max normalization entries per LLM batch (default: 5).	None
normalization_llm	Any | None	Dedicated LLM model instance for tabular normalization.	None

Returns:

Type	Description
ScinrConfig	ScinrConfig singleton containing active library settings.

Raises:

Type	Description
ConfigurationError	If conflicting settings or invalid URIs are supplied.

Source code in src/scinr/newton/config.py

def configure(
    # LLM
    llm: Any | None = None,
    repair_llm: Any | None = None,
    # Neo4j
    neo4j_uri: str | None = None,
    neo4j_user: str | None = None,
    neo4j_password: str | None = None,
    # Models
    enabled_base_themes: list[ThemePath | str] | None = None,
    enabled_user_themes: list[str] | None = None,
    extra_models_paths: list[str | Path] | None = None,
    # Storage
    storage_backend: Literal["none", "mongodb", "custom"] | None = None,
    mongodb_uri: str | None = None,
    mongodb_database: str | None = None,
    mongodb_raw_files_collection: str | None = None,
    mongodb_pages_collection: str | None = None,
    mongodb_gridfs_bucket: str | None = None,
    custom_storage: tuple | None = None,
    # Converters
    extra_converters: dict[str, type] | None = None,
    # PDF
    mistral_api_key: str | None = None,
    # Pipeline behaviour
    prompt_caching_enabled: bool | None = None,
    extraction_batch_size: int | None = None,
    llm_concurrency: int | None = None,
    neo4j_concurrency: int | None = None,
    # Logging
    log_level: str = "INFO",
    # Prompt family
    prompt_family: PromptFamily | Literal["generic", "claude", "gpt_reasoning"] | None = None,
    # Normalization
    normalization_enabled: bool | None = None,
    normalization_batch_size: int | None = None,
    normalization_llm: Any | None = None,
) -> ScinrConfig:
    """
    Configure the scinr-ingest library.

    Parameter resolution order: explicit argument > environment variable > default.

    Args:
        llm: LangChain BaseChatModel instance to use for all LLM calls.
        repair_llm: LangChain BaseChatModel for the JSON repair loop. Falls back to `llm` if None.
        neo4j_uri: Neo4j connection URI. Env: `NEO4J_URI`. Default: `bolt://localhost:7687`.
        neo4j_user: Neo4j username. Env: `NEO4J_USER`.
        neo4j_password: Neo4j password. Env: `NEO4J_PASSWORD`.
        enabled_base_themes: Whitelist of built-in theme paths to activate (`ThemePath` values).
        enabled_user_themes: Whitelist of user theme paths to activate.
        extra_models_paths: Filesystem paths to scan for additional user-defined theme models.
        storage_backend: Storage type: `'none'` (default), `'mongodb'`, or `'custom'`.
        mongodb_uri: MongoDB connection URI. Env: `MONGODB_URI`.
        mongodb_database: MongoDB database name. Env: `MONGODB_DATABASE`.
        mongodb_raw_files_collection: Collection for raw file metadata.
        mongodb_pages_collection: Collection for converted pages.
        mongodb_gridfs_bucket: GridFS bucket name for binary files.
        custom_storage: Tuple `(RawFileRepository, PageRepository)` when `storage_backend='custom'`.
        extra_converters: Dict mapping file extensions to custom `BaseConverter` subclasses.
        mistral_api_key: Mistral API key for PDF OCR conversion.
        prompt_caching_enabled: Enable prompt caching for supported LLM providers.
        extraction_batch_size: Pages per extraction chunk (default: `1`).
        llm_concurrency: Max concurrent LLM calls (semaphore size, default: `4`).
        neo4j_concurrency: Max concurrent Neo4j write sessions (default: `10`).
        log_level: Logging level string (`"DEBUG"`, `"INFO"`, `"WARNING"`, `"ERROR"`).
        prompt_family: Prompt family to use (`"generic"`, `"claude"`, or `"gpt_reasoning"`).
        normalization_enabled: Enable post-extraction normalization for tabular data.
        normalization_batch_size: Max normalization entries per LLM batch (default: `5`).
        normalization_llm: Dedicated LLM model instance for tabular normalization.

    Returns:
        ScinrConfig singleton containing active library settings.

    Raises:
        ConfigurationError: If conflicting settings or invalid URIs are supplied.
    """
    global _config

    # Setup logging first
    logging.basicConfig(level=getattr(logging, log_level.upper(), logging.INFO))

    # ── LLM ──────────────────────────────────────────────────────────────────
    resolved_llm = llm
    if resolved_llm is None:
        model_id = os.getenv("MODEL_ID")
        if model_id:
            try:
                from langchain_aws import ChatBedrockConverse
            except ImportError as exc:
                raise ConfigurationError(
                    "MODEL_ID is set but langchain-aws is not installed.\n"
                    "Install the Bedrock extra: pip install 'scinr-ingest[bedrock]'"
                ) from exc
            resolved_llm = ChatBedrockConverse(
                model=model_id,
                region_name=os.getenv("AWS_DEFAULT_REGION", "us-east-1"),
                max_tokens=int(os.getenv("MAX_TOKENS", "65536")),
                temperature=0,
            )
        else:
            raise ConfigurationError(
                "No LLM configured. Options:\n"
                "\n"
                "  Option 1 — Use any LangChain model:\n"
                "    from langchain_openai import ChatOpenAI\n"
                "    from scinr.newton.config import configure\n"
                "    configure(llm=ChatOpenAI(model='gpt-4o'))\n"
                "\n"
                "  Option 2 — Use AWS Bedrock (define in .env or environment):\n"
                "    MODEL_ID=us.anthropic.claude-sonnet-4-6\n"
                "    AWS_DEFAULT_REGION=us-east-1\n"
            )

    _validate_llm(resolved_llm)

    resolved_repair_llm = repair_llm
    if resolved_repair_llm is None:
        log.warning(
            "Configure -- Specific Repair LLM has not been defined, fallback to main LLM. It is recommended to use a smaller or cheaper model for reparation steps."
        )
        resolved_repair_llm = resolved_llm  # fall back to main LLM

    # ── Neo4j ─────────────────────────────────────────────────────────────────
    resolved_neo4j_uri = neo4j_uri or os.getenv("NEO4J_URI", "bolt://localhost:7687")
    resolved_neo4j_user = neo4j_user or os.getenv("NEO4J_USER")
    resolved_neo4j_password = neo4j_password or os.getenv("NEO4J_PASSWORD")

    # Parse NEO4J_AUTH fallback ("user/password")
    if not resolved_neo4j_user or not resolved_neo4j_password:
        auth_raw = os.getenv("NEO4J_AUTH")
        if auth_raw and "/" in auth_raw:
            parts = auth_raw.split("/", maxsplit=1)
            resolved_neo4j_user = resolved_neo4j_user or parts[0]
            resolved_neo4j_password = resolved_neo4j_password or parts[1]

    if not resolved_neo4j_user:
        raise ConfigurationError(
            "Neo4j username is not configured. Options:\n"
            "  - Set NEO4J_USER=neo4j in your .env file\n"
            "  - Pass neo4j_user='neo4j' to configure()\n"
            "  - Set NEO4J_AUTH=neo4j/password in your .env file"
        )
    if not resolved_neo4j_password:
        raise ConfigurationError(
            "Neo4j password is not configured. Options:\n"
            "  - Set NEO4J_PASSWORD=your_password in your .env file\n"
            "  - Pass neo4j_password='...' to configure()\n"
            "  - Set NEO4J_AUTH=neo4j/password in your .env file"
        )

    # ── Storage ───────────────────────────────────────────────────────────────
    resolved_storage_backend = storage_backend or os.getenv("STORAGE_BACKEND", "none")
    if resolved_storage_backend not in ("none", "mongodb", "custom"):
        raise ConfigurationError(
            f"Unknown storage_backend: {resolved_storage_backend!r}. "
            f"Valid values: 'none', 'mongodb', 'custom'."
        )

    # ── Pipeline behaviour ────────────────────────────────────────────────────
    resolved_caching = (
        prompt_caching_enabled
        if prompt_caching_enabled is not None
        else os.getenv("PROMPT_CACHING_ENABLED", "true").lower() == "true"
    )
    resolved_batch_size = (
        extraction_batch_size
        if extraction_batch_size is not None
        else int(os.getenv("EXTRACTION_BATCH_SIZE", "1"))
    )
    _concurrency_env = os.getenv("LLM_CONCURRENCY", "4")
    resolved_concurrency = llm_concurrency if llm_concurrency is not None else int(_concurrency_env)

    _neo4j_concurrency_env = os.getenv("NEO4J_CONCURRENCY", "10")
    resolved_neo4j_concurrency = neo4j_concurrency if neo4j_concurrency is not None else int(_neo4j_concurrency_env)

    # ── Prompt family ─────────────────────────────────────────────────────────
    _env_prompt_family = os.getenv("PROMPT_FAMILY", "generic").lower()
    if prompt_family is not None:
        try:
            resolved_prompt_family = PromptFamily(prompt_family)
        except ValueError:
            valid = [m.value for m in PromptFamily]
            raise ConfigurationError(
                f"Invalid prompt_family: {prompt_family!r}. "
                f"Valid values: {valid}"
            ) from None
    else:
        try:
            resolved_prompt_family = PromptFamily(_env_prompt_family)
        except ValueError:
            resolved_prompt_family = PromptFamily.GENERIC
            log.warning(
                "Unknown PROMPT_FAMILY env value %r, defaulting to 'generic'.",
                _env_prompt_family,
            )

    # ── Normalization ─────────────────────────────────────────────────────────
    resolved_normalization_enabled = (
        normalization_enabled
        if normalization_enabled is not None
        else os.getenv("NORMALIZATION_ENABLED", "false").lower() == "true"
    )
    resolved_normalization_batch_size = (
        normalization_batch_size
        if normalization_batch_size is not None
        else int(os.getenv("NORMALIZATION_BATCH_SIZE", "5"))
    )
    resolved_normalization_llm = normalization_llm  # Can be None — engine uses main llm as fallback

    # ── Resolve extra_models_paths ────────────────────────────────────────────
    resolved_extra_models_paths: list[Path] = []
    if extra_models_paths:
        resolved_extra_models_paths = [Path(p) for p in extra_models_paths]
    else:
        env_paths = os.getenv("SCINR_EXTRA_MODELS_PATHS", "")
        if env_paths.strip():
            resolved_extra_models_paths = [
                Path(p.strip()) for p in env_paths.split(":") if p.strip()
            ]

    # ── Build config ──────────────────────────────────────────────────────────
    _config = ScinrConfig(
        llm=resolved_llm,
        repair_llm=resolved_repair_llm,
        neo4j_uri=resolved_neo4j_uri,
        neo4j_user=resolved_neo4j_user,
        neo4j_password=resolved_neo4j_password,
        enabled_base_themes=enabled_base_themes,
        enabled_user_themes=enabled_user_themes,
        extra_models_paths=resolved_extra_models_paths,
        storage_backend=resolved_storage_backend,
        mongodb_uri=mongodb_uri or os.getenv("MONGODB_URI", "mongodb://localhost:27017"),
        mongodb_database=mongodb_database or os.getenv("MONGODB_DATABASE", "scinr"),
        mongodb_raw_files_collection=(
            mongodb_raw_files_collection or os.getenv("MONGODB_RAW_FILES_COLLECTION", "raw_files")
        ),
        mongodb_pages_collection=(
            mongodb_pages_collection or os.getenv("MONGODB_PAGES_COLLECTION", "converted_pages")
        ),
        mongodb_gridfs_bucket=(
            mongodb_gridfs_bucket or os.getenv("MONGODB_GRIDFS_BUCKET", "raw_binaries")
        ),
        custom_storage=custom_storage,
        extra_converters=extra_converters or {},
        mistral_api_key=mistral_api_key or os.getenv("MISTRAL_API_KEY"),
        prompt_caching_enabled=resolved_caching,
        extraction_batch_size=resolved_batch_size,
        llm_concurrency=resolved_concurrency,
        neo4j_concurrency=resolved_neo4j_concurrency,
        log_level=log_level,
        prompt_family=resolved_prompt_family,
        normalization_enabled=resolved_normalization_enabled,
        normalization_batch_size=resolved_normalization_batch_size,
        normalization_llm=resolved_normalization_llm,
    )

    # ── Post-init: apply converter overrides ──────────────────────────────────
    if _config.extra_converters:
        from scinr.newton.converters.registry import apply_converter_overrides

        apply_converter_overrides(_config.extra_converters)

    # ── Reset lazy singletons that depend on config ───────────────────────────
    try:
        from scinr.newton.utils.theme_registry import reset_theme_registry

        reset_theme_registry()
    except Exception:
        pass  # theme_registry may not be initialized yet

    try:
        from scinr.newton.ingest.config import _reset_async_driver_singleton
        _reset_async_driver_singleton()
    except Exception:
        pass  # ingest module may not be initialized yet

    try:
        from scinr.newton.storage.mongodb.client import reset_client
        reset_client()
    except Exception:
        pass  # storage mongodb module may not be initialized yet

    log.debug(
        "scinr-ingest configured: storage=%s, llm_concurrency=%d, neo4j_concurrency=%d",
        resolved_storage_backend,
        resolved_concurrency,
        resolved_neo4j_concurrency,
    )
    return _config

scinr.newton.config.get_config

get_config()

Return the current ScinrConfig singleton.

Raises

ConfigurationError If configure() has not been called yet.

Source code in src/scinr/newton/config.py

def get_config() -> ScinrConfig:
    """
    Return the current ScinrConfig singleton.

    Raises
    ------
    ConfigurationError
        If configure() has not been called yet.
    """
    global _config
    if _config is None:
        raise ConfigurationError(
            "scinr-ingest has not been configured yet.\n"
            "Call configure() before using any pipeline function:\n"
            "\n"
            "  from scinr.newton import configure\n"
            "  configure(llm=your_llm, neo4j_uri=..., neo4j_user=..., neo4j_password=...)\n"
            "\n"
            "For CLI usage with environment variables, scinr-ingest handles this automatically."
        )
    return _config  # type: ignore[return-value]

scinr.newton.config.get_available_themes

get_available_themes()

Return all currently registered theme paths, grouped by origin.

Does not require :func:configure to have been called first — it initialises the registry with default settings if needed.

Returns

dict with two keys:

"builtin" Theme paths that ship with the scinr-ingest package. "user" Theme paths loaded from extra_models_paths.

Example::

from scinr.newton import get_available_themes

themes = get_available_themes()
print(themes["builtin"])   # ['default', 'equipment_qualification', ...]
print(themes["user"])      # ['my_custom_theme', ...]

Source code in src/scinr/newton/config.py

def get_available_themes() -> dict[str, list[str]]:
    """
    Return all currently registered theme paths, grouped by origin.

    Does **not** require :func:`configure` to have been called first —
    it initialises the registry with default settings if needed.

    Returns
    -------
    dict with two keys:

    ``"builtin"``
        Theme paths that ship with the scinr-ingest package.
    ``"user"``
        Theme paths loaded from ``extra_models_paths``.

    Example::

        from scinr.newton import get_available_themes

        themes = get_available_themes()
        print(themes["builtin"])   # ['default', 'equipment_qualification', ...]
        print(themes["user"])      # ['my_custom_theme', ...]
    """
    from scinr.newton.utils.theme_registry import get_theme_registry

    registry = get_theme_registry()
    all_paths = registry.get_all_theme_paths()
    user_paths = registry._user_theme_paths
    return {
        "builtin": sorted(p for p in all_paths if p not in user_paths),
        "user": sorted(p for p in all_paths if p in user_paths),
    }

scinr.newton.config.ThemePath module-attribute

ThemePath = Literal[
    "default",
    "equipment_qualification",
    "pharmaceutical_quality",
    "structural_specs",
    "pharma_operations",
    "pharma_operations/product_master",
    "pharma_operations/commercial_sales",
    "pharma_operations/regulatory_portfolio",
    "pharma_operations/batch_manufacturing",
    "pharma_operations/serialization",
    "pharma_regulatory/qa",
    "pharma_regulatory/bpg",
    "pharma_regulatory/variation_guidelines",
]

Theme path identifiers for the built-in scinr-ingest theme library.

Use these values in enabled_base_themes to activate specific built-in themes. Extend with plain str values for user-defined themes added via extra_models_paths.

Example::

configure(
    llm=...,
    enabled_base_themes=["pharmaceutical_quality", "pharma_operations/batch_manufacturing"],
)