graphrag/tests/unit/indexing/input/test_json_loader.py

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License

from graphrag.config.enums import InputFileType
from graphrag.config.models.input_config import InputConfig
from graphrag.index.input.factory import InputReaderFactory
from graphrag_storage import StorageConfig, create_storage


async def test_json_loader_one_file_one_object():
    config = InputConfig(
        storage=StorageConfig(
            base_dir="tests/unit/indexing/input/data/one-json-one-object",
        ),
        file_type=InputFileType.json,
        file_pattern=".*\\.json$",
    )
    storage = create_storage(config.storage)
    documents = (
        await InputReaderFactory()
        .create(config.file_type, {"storage": storage, "config": config})
        .read_files()
    )
    assert documents.shape == (1, 4)
    assert documents["title"].iloc[0] == "input.json"


async def test_json_loader_one_file_multiple_objects():
    config = InputConfig(
        storage=StorageConfig(
            base_dir="tests/unit/indexing/input/data/one-json-multiple-objects",
        ),
        file_type=InputFileType.json,
        file_pattern=".*\\.json$",
    )
    storage = create_storage(config.storage)
    documents = (
        await InputReaderFactory()
        .create(config.file_type, {"storage": storage, "config": config})
        .read_files()
    )
    print(documents)
    assert documents.shape == (3, 4)
    assert documents["title"].iloc[0] == "input.json"


async def test_json_loader_one_file_with_title():
    config = InputConfig(
        storage=StorageConfig(
            base_dir="tests/unit/indexing/input/data/one-json-one-object",
        ),
        file_type=InputFileType.json,
        file_pattern=".*\\.json$",
        title_column="title",
    )
    storage = create_storage(config.storage)
    documents = (
        await InputReaderFactory()
        .create(config.file_type, {"storage": storage, "config": config})
        .read_files()
    )
    assert documents.shape == (1, 4)
    assert documents["title"].iloc[0] == "Hello"


async def test_json_loader_one_file_with_metadata():
    config = InputConfig(
        storage=StorageConfig(
            base_dir="tests/unit/indexing/input/data/one-json-one-object",
        ),
        file_type=InputFileType.json,
        file_pattern=".*\\.json$",
        title_column="title",
        metadata=["title"],
    )
    storage = create_storage(config.storage)
    documents = (
        await InputReaderFactory()
        .create(config.file_type, {"storage": storage, "config": config})
        .read_files()
    )
    assert documents.shape == (1, 5)
    assert documents["metadata"][0] == {"title": "Hello"}


async def test_json_loader_multiple_files():
    config = InputConfig(
        storage=StorageConfig(
            base_dir="tests/unit/indexing/input/data/multiple-jsons",
        ),
        file_type=InputFileType.json,
        file_pattern=".*\\.json$",
    )
    storage = create_storage(config.storage)
    documents = (
        await InputReaderFactory()
        .create(config.file_type, {"storage": storage, "config": config})
        .read_files()
    )
    assert documents.shape == (4, 4)