mirror of
https://github.com/microsoft/graphrag.git
synced 2026-01-29 23:31:51 +08:00
1df89727c3
Some checks failed
gh-pages / build (push) Has been cancelled
Python CI / python-ci (ubuntu-latest, 3.10) (push) Has been cancelled
Python CI / python-ci (ubuntu-latest, 3.11) (push) Has been cancelled
Python CI / python-ci (windows-latest, 3.10) (push) Has been cancelled
Python CI / python-ci (windows-latest, 3.11) (push) Has been cancelled
Python Integration Tests / python-ci (ubuntu-latest, 3.10) (push) Has been cancelled
Python Integration Tests / python-ci (windows-latest, 3.10) (push) Has been cancelled
Python Notebook Tests / python-ci (ubuntu-latest, 3.10) (push) Has been cancelled
Python Notebook Tests / python-ci (windows-latest, 3.10) (push) Has been cancelled
Python Publish (pypi) / Upload release to PyPI (push) Has been cancelled
Python Smoke Tests / python-ci (ubuntu-latest, 3.10) (push) Has been cancelled
Python Smoke Tests / python-ci (windows-latest, 3.10) (push) Has been cancelled
Spellcheck / spellcheck (push) Has been cancelled
* Move covariate run conditional * All pipeline registration * Fix method name construction * Rename context storage -> output_storage * Rename OutputConfig as generic StorageConfig * Reuse Storage model under InputConfig * Move input storage creation out of document loading * Move document loading into workflows * Semver * Fix smoke test config for new workflows * Fix unit tests --------- Co-authored-by: Alonso Guevara <alonsog@microsoft.com>
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
# Copyright (c) 2024 Microsoft Corporation.
|
|
# Licensed under the MIT License
|
|
|
|
from graphrag.config.enums import InputFileType
|
|
from graphrag.config.models.input_config import InputConfig
|
|
from graphrag.config.models.storage_config import StorageConfig
|
|
from graphrag.index.input.factory import create_input
|
|
from graphrag.utils.api import create_storage_from_config
|
|
|
|
|
|
async def test_csv_loader_one_file():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-csv",
|
|
),
|
|
file_type=InputFileType.csv,
|
|
file_pattern=".*\\.csv$",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (2, 4)
|
|
assert documents["title"].iloc[0] == "input.csv"
|
|
|
|
|
|
async def test_csv_loader_one_file_with_title():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-csv",
|
|
),
|
|
file_type=InputFileType.csv,
|
|
file_pattern=".*\\.csv$",
|
|
title_column="title",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (2, 4)
|
|
assert documents["title"].iloc[0] == "Hello"
|
|
|
|
|
|
async def test_csv_loader_one_file_with_metadata():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/one-csv",
|
|
),
|
|
file_type=InputFileType.csv,
|
|
file_pattern=".*\\.csv$",
|
|
title_column="title",
|
|
metadata=["title"],
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (2, 5)
|
|
assert documents["metadata"][0] == {"title": "Hello"}
|
|
|
|
|
|
async def test_csv_loader_multiple_files():
|
|
config = InputConfig(
|
|
storage=StorageConfig(
|
|
base_dir="tests/unit/indexing/input/data/multiple-csvs",
|
|
),
|
|
file_type=InputFileType.csv,
|
|
file_pattern=".*\\.csv$",
|
|
)
|
|
storage = create_storage_from_config(config.storage)
|
|
documents = await create_input(config=config, storage=storage)
|
|
assert documents.shape == (4, 4)
|