Global search with dynamic community selection
+ + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [1]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+# Copyright (c) 2024 Microsoft Corporation.
+# Licensed under the MIT License.
+# Copyright (c) 2024 Microsoft Corporation.
+# Licensed under the MIT License.
+
+
+
+
+
+
+
+
+
+
+In [2]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+import os
+
+import pandas as pd
+import tiktoken
+
+from graphrag.query.indexer_adapters import (
+ read_indexer_communities,
+ read_indexer_entities,
+ read_indexer_reports,
+)
+from graphrag.query.llm.oai.chat_openai import ChatOpenAI
+from graphrag.query.llm.oai.typing import OpenaiApiType
+from graphrag.query.structured_search.global_search.community_context import (
+ GlobalCommunityContext,
+)
+from graphrag.query.structured_search.global_search.search import GlobalSearch
+import os
+
+import pandas as pd
+import tiktoken
+
+from graphrag.query.indexer_adapters import (
+ read_indexer_communities,
+ read_indexer_entities,
+ read_indexer_reports,
+)
+from graphrag.query.llm.oai.chat_openai import ChatOpenAI
+from graphrag.query.llm.oai.typing import OpenaiApiType
+from graphrag.query.structured_search.global_search.community_context import (
+ GlobalCommunityContext,
+)
+from graphrag.query.structured_search.global_search.search import GlobalSearch
+
+
+
+
+
+
+
+
+
+
+Global Search example¶
Global search method generates answers by searching over all AI-generated community reports in a map-reduce fashion. This is a resource-intensive method, but often gives good responses for questions that require an understanding of the dataset as a whole (e.g. What are the most significant values of the herbs mentioned in this notebook?).
+
+
+
+
+
+
+
+
+
+LLM setup¶
+
+
+
+
+
+
+
+
+
+
+In [3]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+api_key = os.environ["GRAPHRAG_API_KEY"]
+llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
+
+llm = ChatOpenAI(
+ api_key=api_key,
+ model=llm_model,
+ api_type=OpenaiApiType.OpenAI, # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI
+ max_retries=20,
+)
+
+token_encoder = tiktoken.encoding_for_model(llm_model)
+api_key = os.environ["GRAPHRAG_API_KEY"]
+llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
+
+llm = ChatOpenAI(
+ api_key=api_key,
+ model=llm_model,
+ api_type=OpenaiApiType.OpenAI, # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI
+ max_retries=20,
+)
+
+token_encoder = tiktoken.encoding_for_model(llm_model)
+
+
+
+
+
+
+
+
+
+Load community reports as context for global search¶
-
+
- Load all community reports in the
create_final_community_reportstable from the ire-indexing engine, to be used as context data for global search.
+ - Load entities from the
create_final_nodesandcreate_final_entitiestables from the ire-indexing engine, to be used for calculating community weights for context ranking. Note that this is optional (if no entities are provided, we will not calculate community weights and only use the rank attribute in the community reports table for context ranking)
+ - Load all communities in the
create_final_communitestable from the ire-indexing engine, to be used to reconstruct the community graph hierarchy for dynamic community selection.
+
+
+
+
+
+
+
+
+
+In [4]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+# parquet files generated from indexing pipeline
+INPUT_DIR = "./inputs/operation dulce"
+COMMUNITY_TABLE = "create_final_communities"
+COMMUNITY_REPORT_TABLE = "create_final_community_reports"
+ENTITY_TABLE = "create_final_nodes"
+ENTITY_EMBEDDING_TABLE = "create_final_entities"
+
+# we don't fix a specific community level but instead use an agent to dynamicially
+# search through all the community reports to check if they are relevant.
+COMMUNITY_LEVEL = None
+# parquet files generated from indexing pipeline
+INPUT_DIR = "./inputs/operation dulce"
+COMMUNITY_TABLE = "create_final_communities"
+COMMUNITY_REPORT_TABLE = "create_final_community_reports"
+ENTITY_TABLE = "create_final_nodes"
+ENTITY_EMBEDDING_TABLE = "create_final_entities"
+
+# we don't fix a specific community level but instead use an agent to dynamicially
+# search through all the community reports to check if they are relevant.
+COMMUNITY_LEVEL = None
+
+
+
+
+
+
+
+
+
+
+In [5]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")
+entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
+report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
+entity_embedding_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_EMBEDDING_TABLE}.parquet")
+
+communities = read_indexer_communities(community_df, entity_df, report_df)
+reports = read_indexer_reports(
+ report_df,
+ entity_df,
+ community_level=COMMUNITY_LEVEL,
+ dynamic_community_selection=True,
+)
+entities = read_indexer_entities(
+ entity_df, entity_embedding_df, community_level=COMMUNITY_LEVEL
+)
+
+print(f"Total report count: {len(report_df)}")
+print(
+ f"Report count after filtering by community level {COMMUNITY_LEVEL}: {len(reports)}"
+)
+
+report_df.head()
+community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")
+entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
+report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
+entity_embedding_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_EMBEDDING_TABLE}.parquet")
+
+communities = read_indexer_communities(community_df, entity_df, report_df)
+reports = read_indexer_reports(
+ report_df,
+ entity_df,
+ community_level=COMMUNITY_LEVEL,
+ dynamic_community_selection=True,
+)
+entities = read_indexer_entities(
+ entity_df, entity_embedding_df, community_level=COMMUNITY_LEVEL
+)
+
+print(f"Total report count: {len(report_df)}")
+print(
+ f"Report count after filtering by community level {COMMUNITY_LEVEL}: {len(reports)}"
+)
+
+report_df.head()
+
+
+
+
+
+
+
+
+
+
+
+Total report count: 20 +Report count after filtering by community level None: 20 ++
+
+Out[5]:
+
+
+
+
+
+
+
+
+| + | community | +full_content | +level | +rank | +title | +rank_explanation | +summary | +findings | +full_content_json | +id | +
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | +10 | +# Paranormal Military Squad at Dulce Base: Dec... | +1 | +8.5 | +Paranormal Military Squad at Dulce Base: Decod... | +The impact severity rating is high due to the ... | +The Paranormal Military Squad, stationed at Du... | +[{'explanation': 'Jordan is a central figure i... | +{\n "title": "Paranormal Military Squad at ... | +1ba2d200-dd26-4693-affe-a5539d0a0e0d | +
| 1 | +11 | +# Dulce and Paranormal Military Squad Operatio... | +1 | +8.5 | +Dulce and Paranormal Military Squad Operations | +The impact severity rating is high due to the ... | +The community centers around Dulce, a secretiv... | +[{'explanation': 'Dulce is described as a top-... | +{\n "title": "Dulce and Paranormal Military... | +a8a530b0-ae6b-44ea-b11c-9f70d138298d | +
| 2 | +12 | +# Paranormal Military Squad and Dulce Base Ope... | +1 | +7.5 | +Paranormal Military Squad and Dulce Base Opera... | +The impact severity rating is relatively high ... | +The community centers around the Paranormal Mi... | +[{'explanation': 'Taylor is a central figure w... | +{\n "title": "Paranormal Military Squad and... | +0478975b-c805-4cc1-b746-82f3e689e2f3 | +
| 3 | +13 | +# Mission Dynamics and Leadership: Cruz and Wa... | +1 | +7.5 | +Mission Dynamics and Leadership: Cruz and Wash... | +The impact severity rating is relatively high ... | +This report explores the intricate dynamics of... | +[{'explanation': 'Cruz is a central figure in ... | +{\n "title": "Mission Dynamics and Leadersh... | +b56f6e68-3951-4f07-8760-63700944a375 | +
| 4 | +14 | +# Dulce Base and Paranormal Military Squad: Br... | +1 | +8.5 | +Dulce Base and Paranormal Military Squad: Brid... | +The impact severity rating is high due to the ... | +The community centers around the Dulce Base, a... | +[{'explanation': 'Sam Rivera, a member of the ... | +{\n "title": "Dulce Base and Paranormal Mil... | +736e7006-d050-4abb-a122-00febf3f540f | +
+
+
+
+
+
+
+
+
+Build global context with dynamic community selection¶
The goal of dynamic community selection reduce the number of community reports that need to be processed in the map-reduce operation. To that end, we take advantage of the hierachical structure of the indexed dataset. We first ask the LLM to rate how relevant each level 0 community is with respect to the user query, we then traverse down the child node(s) if the current community report is deemed relevant.
+You can still set a COMMUNITY_LEVEL to filter out lower level community reports and apply dynamic community selection on the filtered reports.
Note that the dataset is quite small, with only consist of 20 communities from 2 levels (level 0 and 1). Dynamic community selection is more effective when there are large amount of content to be filtered out.
+
+
+
+
+
+
+
+
+
+
+In [6]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+mini_llm = ChatOpenAI(
+ api_key=api_key,
+ model="gpt-4o-mini",
+ api_type=OpenaiApiType.OpenAI, # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI
+ max_retries=20,
+)
+mini_token_encoder = tiktoken.encoding_for_model(mini_llm.model)
+
+context_builder = GlobalCommunityContext(
+ community_reports=reports,
+ communities=communities,
+ entities=entities, # default to None if you don't want to use community weights for ranking
+ token_encoder=token_encoder,
+ dynamic_community_selection=True,
+ dynamic_community_selection_kwargs={
+ "llm": mini_llm,
+ "token_encoder": mini_token_encoder,
+ },
+)
+mini_llm = ChatOpenAI(
+ api_key=api_key,
+ model="gpt-4o-mini",
+ api_type=OpenaiApiType.OpenAI, # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI
+ max_retries=20,
+)
+mini_token_encoder = tiktoken.encoding_for_model(mini_llm.model)
+
+context_builder = GlobalCommunityContext(
+ community_reports=reports,
+ communities=communities,
+ entities=entities, # default to None if you don't want to use community weights for ranking
+ token_encoder=token_encoder,
+ dynamic_community_selection=True,
+ dynamic_community_selection_kwargs={
+ "llm": mini_llm,
+ "token_encoder": mini_token_encoder,
+ },
+)
+
+
+
+
+
+
+
+
+
+Perform global search with dynamic community selection¶
+
+
+
+
+
+
+
+
+
+In [7]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+context_builder_params = {
+ "use_community_summary": False, # False means using full community reports. True means using community short summaries.
+ "shuffle_data": True,
+ "include_community_rank": True,
+ "min_community_rank": 0,
+ "community_rank_name": "rank",
+ "include_community_weight": True,
+ "community_weight_name": "occurrence weight",
+ "normalize_community_weight": True,
+ "max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
+ "context_name": "Reports",
+}
+
+map_llm_params = {
+ "max_tokens": 1000,
+ "temperature": 0.0,
+ "response_format": {"type": "json_object"},
+}
+
+reduce_llm_params = {
+ "max_tokens": 2000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000-1500)
+ "temperature": 0.0,
+}
+context_builder_params = {
+ "use_community_summary": False, # False means using full community reports. True means using community short summaries.
+ "shuffle_data": True,
+ "include_community_rank": True,
+ "min_community_rank": 0,
+ "community_rank_name": "rank",
+ "include_community_weight": True,
+ "community_weight_name": "occurrence weight",
+ "normalize_community_weight": True,
+ "max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
+ "context_name": "Reports",
+}
+
+map_llm_params = {
+ "max_tokens": 1000,
+ "temperature": 0.0,
+ "response_format": {"type": "json_object"},
+}
+
+reduce_llm_params = {
+ "max_tokens": 2000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000-1500)
+ "temperature": 0.0,
+}
+
+
+
+
+
+
+
+
+
+In [8]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+search_engine = GlobalSearch(
+ llm=llm,
+ context_builder=context_builder,
+ token_encoder=token_encoder,
+ max_data_tokens=12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
+ map_llm_params=map_llm_params,
+ reduce_llm_params=reduce_llm_params,
+ allow_general_knowledge=False, # set this to True will add instruction to encourage the LLM to incorporate general knowledge in the response, which may increase hallucinations, but could be useful in some use cases.
+ json_mode=True, # set this to False if your LLM model does not support JSON mode.
+ context_builder_params=context_builder_params,
+ concurrent_coroutines=32,
+ response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
+)
+search_engine = GlobalSearch(
+ llm=llm,
+ context_builder=context_builder,
+ token_encoder=token_encoder,
+ max_data_tokens=12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
+ map_llm_params=map_llm_params,
+ reduce_llm_params=reduce_llm_params,
+ allow_general_knowledge=False, # set this to True will add instruction to encourage the LLM to incorporate general knowledge in the response, which may increase hallucinations, but could be useful in some use cases.
+ json_mode=True, # set this to False if your LLM model does not support JSON mode.
+ context_builder_params=context_builder_params,
+ concurrent_coroutines=32,
+ response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
+)
+
+
+
+
+
+
+
+
+
+In [9]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+result = await search_engine.asearch(
+ "What is Cosmic Vocalization and who are involved in it?"
+)
+
+print(result.response)
+result = await search_engine.asearch(
+ "What is Cosmic Vocalization and who are involved in it?"
+)
+
+print(result.response)
+
+
+
+
+
+
+
+
+
+
+
+### Overview of Cosmic Vocalization + +Cosmic Vocalization is a phenomenon that has captured the attention of various individuals and groups within the community. It is perceived as a significant cosmic event, with implications that extend beyond mere observation. The concept is central to the community's focus, suggesting its importance in both cultural and strategic contexts [Data: Reports (6)]. + +### Key Perspectives and Concerns + +Alex Mercer views Cosmic Vocalization as part of an interstellar duet, indicating a responsive and perhaps interactive approach to these cosmic events. This perspective suggests that Cosmic Vocalization may not be a solitary occurrence but part of a larger cosmic interaction [Data: Reports (6)]. + +On the other hand, Taylor Cruz raises concerns about the potential implications of Cosmic Vocalization, fearing it might serve as a homing tune. This adds a layer of urgency and potential threat, as it implies that the phenomenon could attract attention from unknown entities or forces [Data: Reports (6)]. + +### Strategic and Metaphorical Interpretations + +The Paranormal Military Squad's involvement with Cosmic Vocalization highlights its strategic importance. Their engagement suggests that these cosmic phenomena are not only of scientific interest but also of security concern, warranting a structured response as part of their mission [Data: Reports (6)]. + +Furthermore, the Universe is metaphorically treated as a concert hall by the Paranormal Military Squad. This metaphorical interpretation suggests a broader perspective on how cosmic events are perceived and responded to by human entities, emphasizing the cultural and philosophical dimensions of Cosmic Vocalization [Data: Reports (6)]. + +In summary, Cosmic Vocalization is a multifaceted phenomenon involving various stakeholders, each with their own interpretations and concerns. Its significance spans cultural, strategic, and security domains, making it a subject of considerable interest and debate within the community. ++
+
+
+
+
+
+
+
+
+In [10]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+# inspect the data used to build the context for the LLM responses
+result.context_data["reports"]
+# inspect the data used to build the context for the LLM responses
+result.context_data["reports"]
+
+
+
+
+
+
+
+
+Out[10]:
+
+
+
+
+
+
+
+
+| + | id | +title | +occurrence weight | +content | +rank | +
|---|---|---|---|---|---|
| 0 | +15 | +Dulce Base and the Paranormal Military Squad: ... | +1.00 | +# Dulce Base and the Paranormal Military Squad... | +9.5 | +
| 1 | +16 | +Dulce Military Base and Alien Intelligence Com... | +0.08 | +# Dulce Military Base and Alien Intelligence C... | +8.5 | +
| 2 | +18 | +Paranormal Military Squad Team and Dulce Base'... | +0.04 | +# Paranormal Military Squad Team and Dulce Bas... | +8.5 | +
| 3 | +19 | +Central Terminal and Viewing Monitors at Dulce... | +0.02 | +# Central Terminal and Viewing Monitors at Dul... | +8.5 | +
| 4 | +17 | +Dulce Team and Underground Command Center: Int... | +0.02 | +# Dulce Team and Underground Command Center: I... | +8.5 | +
| 5 | +4 | +Dulce Facility and Control Room of Dulce: Extr... | +0.02 | +# Dulce Facility and Control Room of Dulce: Ex... | +8.5 | +
| 6 | +6 | +Cosmic Vocalization and Universe Interactions | +0.02 | +# Cosmic Vocalization and Universe Interaction... | +7.5 | +
+
+
+
+
+
+
+
+
+
+In [11]:
+
+
+
+
+
+
+
+
+
+Copied!
+
+
+# inspect number of LLM calls and tokens in dynamic community selection
+llm_calls = result.llm_calls_categories["build_context"]
+prompt_tokens = result.prompt_tokens_categories["build_context"]
+output_tokens = result.output_tokens_categories["build_context"]
+print(
+ f"Build context ({mini_llm.model})\nLLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
+)
+# inspect number of LLM calls and tokens in map-reduce
+llm_calls = result.llm_calls_categories["map"] + result.llm_calls_categories["reduce"]
+prompt_tokens = (
+ result.prompt_tokens_categories["map"] + result.prompt_tokens_categories["reduce"]
+)
+output_tokens = (
+ result.output_tokens_categories["map"] + result.output_tokens_categories["reduce"]
+)
+print(
+ f"Map-reduce ({llm.model})\nLLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
+)
+# inspect number of LLM calls and tokens in dynamic community selection
+llm_calls = result.llm_calls_categories["build_context"]
+prompt_tokens = result.prompt_tokens_categories["build_context"]
+output_tokens = result.output_tokens_categories["build_context"]
+print(
+ f"Build context ({mini_llm.model})\nLLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
+)
+# inspect number of LLM calls and tokens in map-reduce
+llm_calls = result.llm_calls_categories["map"] + result.llm_calls_categories["reduce"]
+prompt_tokens = (
+ result.prompt_tokens_categories["map"] + result.prompt_tokens_categories["reduce"]
+)
+output_tokens = (
+ result.output_tokens_categories["map"] + result.output_tokens_categories["reduce"]
+)
+print(
+ f"Map-reduce ({llm.model})\nLLM calls: {llm_calls}. Prompt tokens: {prompt_tokens}. Output tokens: {output_tokens}."
+)
+
+
+
+
+
+
+
+
+
+
+
+Build context (gpt-4o-mini) +LLM calls: 12. Prompt tokens: 8565. Output tokens: 1109. +Map-reduce (gpt-4o) +LLM calls: 2. Prompt tokens: 5834. Output tokens: 600. ++