obscura/zydi-lab
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加了文献分析的分段分析功能

Browse Source
This commit is contained in:
zydi
2025-01-15 06:48:41 +00:00
parent d4084afb6b
commit e7311477ff
2 changed files with 317 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
literature.html
+70 -62
View File
@@ -654,70 +654,78 @@
// Get analysis status and results for each literature
references.forEach(async (ref) => {
try {
const analysisResponse = await fetch(`${API_BASE_URL}/references/${ref._id}/report`, {
headers: {
'Authorization': `Bearer ${token}`
}
});
if (!analysisResponse.ok) {
throw new Error('Failed to fetch analysis');
}
const analysisResult = await analysisResponse.json();
const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
const statusElement = document.querySelector(`.analysis-status-${ref._id}`);
if (analysisResult.status === 'processing') {
// Processing status
contentElement.innerHTML = `
<div class="d-flex align-items-center">
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span>Analyzing literature content...</span>
</div>
`;
statusElement.innerHTML = '<span class="text-primary">Analysis in progress...</span>';
} else if (analysisResult.status === 'completed') {
// Completed status
let author = 'N/A';
let publicationDate = 'N/A';
let journal = 'N/A';
try {
if (analysisResult['1. Basic Information']) {
const basicInfo = analysisResult['1. Basic Information'];
author = basicInfo.author || 'N/A';
publicationDate = basicInfo.publication_date || 'N/A';
journal = basicInfo.journal_publisher || 'N/A';
// 更新卡片标题为分析结果中的标题
const titleElement = document.querySelector(`#reference-${ref._id} .card-title`);
if (titleElement && basicInfo.title) {
titleElement.textContent = basicInfo.title;
}
const checkAnalysisStatus = async () => {
const analysisResponse = await fetch(`${API_BASE_URL}/references/${ref._id}/report`, {
headers: {
'Authorization': `Bearer ${token}`
}
contentElement.innerHTML = `
<p class="mb-1">Author: ${author}</p>
<p class="mb-1">Publication Date: ${publicationDate}</p>
<p class="mb-1">Journal/Publisher: ${journal}</p>
`;
statusElement.innerHTML = '<span class="text-success">Analysis completed</span>';
} catch (parseError) {
console.error('Failed to parse analysis result:', parseError);
});
if (!analysisResponse.ok) {
throw new Error('Failed to fetch analysis');
}
} else if (analysisResult.status === 'failed') {
// Failed status
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Analysis failed
</div>
`;
statusElement.innerHTML = `<span class="text-danger">Analysis failed: ${analysisResult.message || 'Unknown error'}</span>`;
}
const analysisResult = await analysisResponse.json();
const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
const statusElement = document.querySelector(`.analysis-status-${ref._id}`);
if (analysisResult.status === 'processing') {
// Processing status
contentElement.innerHTML = `
<div class="d-flex align-items-center">
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span>Analyzing literature content...</span>
</div>
`;
statusElement.innerHTML = '<span class="text-primary">Analysis in progress...</span>';
// 如果还在处理中，5秒后再次检查
setTimeout(checkAnalysisStatus, 10000);
} else if (analysisResult.status === 'completed') {
// Completed status
let author = 'N/A';
let publicationDate = 'N/A';
let journal = 'N/A';
try {
if (analysisResult['1. Basic Information']) {
const basicInfo = analysisResult['1. Basic Information'];
author = basicInfo.author || 'N/A';
publicationDate = basicInfo.publication_date || 'N/A';
journal = basicInfo.journal_publisher || 'N/A';
// 更新卡片标题为分析结果中的标题
const titleElement = document.querySelector(`#reference-${ref._id} .card-title`);
if (titleElement && basicInfo.title) {
titleElement.textContent = basicInfo.title;
}
}
contentElement.innerHTML = `
<p class="mb-1">Author: ${author}</p>
<p class="mb-1">Publication Date: ${publicationDate}</p>
<p class="mb-1">Journal/Publisher: ${journal}</p>
`;
statusElement.innerHTML = '<span class="text-success">Analysis completed</span>';
} catch (parseError) {
console.error('Failed to parse analysis result:', parseError);
}
} else if (analysisResult.status === 'failed') {
// Failed status
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Analysis failed
</div>
`;
statusElement.innerHTML = `<span class="text-danger">Analysis failed: ${analysisResult.message || 'Unknown error'}</span>`;
}
};
// 开始首次检查
await checkAnalysisStatus();
} catch (error) {
console.error(`Failed to fetch analysis for reference ${ref._id}:`, error);
const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
main.py
+247 -122
View File
@@ -1648,71 +1648,7 @@ async def analyze_project_data(project_data):
print(f"Error calling DeepSeek API: {e}")
return None
async def analyze_reference_document_async(content: str):
"""分析文献的基本信息和内容"""
system_prompt = """
You are an AI assistant tasked with analyzing academic literature.
Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
The JSON structure must strictly follow the provided template.
"""
user_prompt = f"""Analyze the following literature and extract all relevant information:
Content: {content}
Generate a JSON response with the following structure:
{{
"1. Basic Information": {{
"author": "[Author name(s) and affiliations]",
"publication_date": "[Publication date in YYYY-MM format]",
"title": "[Full title of the document]",
"journal_publisher": "[Journal name or publisher details]",
"document_type": "[Type: journal article/book/conference paper etc.]"
}},
"2. Content Analysis": {{
"abstract": "[Paper abstract]",
"research_purpose": "[Main objectives and research questions]",
"methodology": "[Research methods, data collection and analysis approaches]",
"main_arguments": "[Key theoretical frameworks and arguments]",
"conclusions": "[Primary findings and conclusions]",
"innovations": "[Novel contributions and original aspects]"
}}
}}
"""
return await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
async def analyze_reference_value_async(content_analysis: dict):
"""基于内容分析结果评估文献的价值"""
system_prompt = """
You are an AI assistant tasked with evaluating the value of academic literature based on its content analysis.
Generate a comprehensive value evaluation in JSON format.
The JSON structure must strictly follow the provided template.
"""
user_prompt = f"""Based on the following content analysis, evaluate the literature's value:
Content Analysis: {json.dumps(content_analysis, ensure_ascii=False)}
Generate a JSON response with the following structure:
{{
"3. Value Evaluation": {{
"academic_contribution": "[Significance to the field of study]",
"practical_significance": "[Real-world applications and implications]",
"limitations": "[Research constraints and weaknesses]",
"implications": "[Suggestions for future research and practice]"
}}
}}
"""
return await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
# 修改函数定义为异步函数
async def analyze_reference_summary(reference_data):
system_prompt = """
@@ -2810,25 +2746,72 @@ async def process_batch_analysis(references: List[dict]):
print(f"Reference file not found: {file_path}")
return
print(f"\n开始处理文献 {ref.get('reference_title', '未知标题')}")
print(f"文献ID: {reference_id}")
print(f"文件路径: {file_path}")
# 异步读取PDF
print("\n=== 步骤1: 读取PDF文件 ===")
pdf_content = await read_pdf_async(file_path)
if not pdf_content:
raise Exception("Failed to read PDF content")
# 异步分析文档
document_analysis = await analyze_reference_document_async(pdf_content)
if not document_analysis:
raise Exception("Failed to analyze document")
# 打印字符数
content_length = len(pdf_content)
print(f"\n=== 步骤2: 内容长度检查 ===")
print(f"PDF内容总字符数: {content_length}")
# 根据内容长度选择不同的处理方式
if content_length <= 200000:
print(f"\n=== 步骤3A: 使用直接分析方式 ===")
print(f"文档长度在处理范围内 ({content_length} <= 200000)")
# 直接分析文档内容
print("开始分析文档内容...")
document_analysis = await analyze_reference_document_async(pdf_content[:180000])
if not document_analysis:
raise Exception("Failed to analyze document")
print("文档分析完成")
else:
print(f"\n=== 步骤3B: 使用分段分析方式 ===")
print(f"文档超过200000字符 ({content_length} > 200000)")
# 分段分析长文档
print("\n--- 开始分段分析 ---")
print("正在调用 analyze_long_document_async...")
try:
analysis_results = await analyze_long_document_async(pdf_content)
except Exception as e:
print(f"分段分析过程中出错: {str(e)}")
raise
if not analysis_results:
raise Exception("Failed to analyze document in segments")
print(f"分段分析完成，共分析了 {len(analysis_results)} 个段落")
# 合并分析结果
try:
document_analysis = await merge_analysis_results(analysis_results)
except Exception as e:
raise
if not document_analysis:
raise Exception("Failed to merge analysis results")
# 等待一小段时间避免API限制
await asyncio.sleep(1)
# 异步分析价值
value_evaluation = await analyze_reference_value_async(document_analysis)
# 异步分析文献价值
try:
value_evaluation = await analyze_reference_value_async(document_analysis)
print(f"analyze_reference_value_async 返回结果类型: {type(value_evaluation)}")
except Exception as e:
raise
if not value_evaluation:
raise Exception("Failed to evaluate value")
print("文献价值分析完成")
# 合并结果
print("\n=== 步骤5: 保存最终结果 ===")
analysis_result = {
**document_analysis,
**value_evaluation,
@@ -2841,7 +2824,6 @@ async def process_batch_analysis(references: List[dict]):
await redis.set(report_key, json.dumps(analysis_result))
except Exception as e:
print(f"Error processing reference {ref['reference_id']}: {str(e)}")
try:
await redis.select(203)
report_key = f"reference_report:{ref['reference_id']}"
@@ -2884,7 +2866,21 @@ async def read_pdf_async(file_path: str) -> str:
for page in pdf_reader.pages:
content = page.extract_text()
pdf_content += content
return pdf_content[:180000] # 限制内容长度
# 根据内容长度返回不同的结果
content_length = len(pdf_content)
print(f"\nPDF原始内容字符数: {content_length}")
if content_length <= 180000:
print("文档长度 ≤ 180000，返回完整内容")
return pdf_content
elif content_length <= 200000:
print("文档长度在 180000-200000 之间，截取前 180000 个字符")
return pdf_content[:180000]
else:
print("文档长度 > 200000，返回完整内容供分段处理")
return pdf_content # 返回完整内容，由调用者处理分段
except Exception as e:
print(f"Error reading PDF: {e}")
return ""
@@ -2906,62 +2902,191 @@ async def call_deepseek_api_async(messages: list) -> dict:
else:
raise Exception(f"API调用失败: {await response.text()}")
async def process_reference_analysis(reference_id: str, reference: dict):
"""后台处理文献分析"""
redis = await get_redis()
async def analyze_long_document_async(content: str) -> List[dict]:
"""分段分析长文档"""
# 将内容分成多个段落，每段约60000个字符（估算后约50000 tokens）
segments = []
content_length = len(content)
segment_size = 60000 # 减小段落大小
try:
file_path = reference.get("reference_link")
if not file_path or not os.path.exists(file_path):
print(f"Reference file not found: {file_path}")
return
# 异步读取PDF
pdf_content = await read_pdf_async(file_path)
if not pdf_content:
raise Exception("Failed to read PDF content")
print(f"\n开始分段处理，总字符数: {content_length}")
print(f"每段大小: {segment_size} 字符")
for i in range(0, content_length, segment_size):
segment = content[i:i + segment_size]
segments.append(segment)
print(f"文档已分段，共 {len(segments)} 个段落")
# 对每个段落进行分析
analysis_results = []
for i, segment in enumerate(segments):
print(f"\n开始分析第 {i+1}/{len(segments)} 个段落...")
# 异步分析文档内容
document_analysis = await analyze_reference_document_async(pdf_content)
if not document_analysis:
print("Failed to analyze document")
return
# 异步分析文献价值
value_evaluation = await analyze_reference_value_async(document_analysis)
if not value_evaluation:
print("Failed to evaluate value")
return
# 合并分析结果
analysis_result = {
**document_analysis,
**value_evaluation
}
system_prompt = f"""
You are an AI assistant tasked with analyzing part {i+1} of {len(segments)} of an academic literature.
Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
Note that this is part {i+1} of a longer document, so focus on the content provided.
The JSON structure must strictly follow the provided template.
"""
user_prompt = f"""Analyze the following literature segment and extract all relevant information:
Content: {segment}
# 保存分析结果到db203
await redis.select(203)
report_key = f"reference_report:{reference_id}"
await redis.set(report_key, json.dumps(analysis_result))
Generate a JSON response with the following structure:
{{
"1. Basic Information": {{
"author": "[Author name(s) and affiliations]",
"publication_date": "[Publication date in YYYY-MM format]",
"title": "[Full title of the document]",
"journal_publisher": "[Journal name or publisher details]",
"document_type": "[Type: journal article/book/conference paper etc.]"
}},
"2. Content Analysis": {{
"abstract": "[Paper abstract]",
"research_purpose": "[Main objectives and research questions]",
"methodology": "[Research methods, data collection and analysis approaches]",
"main_arguments": "[Key theoretical frameworks and arguments]",
"conclusions": "[Primary findings and conclusions]",
"innovations": "[Novel contributions and original aspects]"
}}
}}
"""
except Exception as e:
print(f"Error analyzing reference data: {str(e)}")
try:
await redis.select(203)
report_key = f"reference_report:{reference_id}"
error_status = {
"status": "failed",
"message": str(e)
}
await redis.set(report_key, json.dumps(error_status))
except Exception as redis_error:
print(f"Error updating Redis status: {redis_error}")
finally:
try:
await redis.aclose()
result = await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
if result:
print(f"第 {i+1} 个段落分析完成")
analysis_results.append(result)
# 等待一小段时间避免API限制
await asyncio.sleep(1)
else:
print(f"第 {i+1} 个段落分析失败")
except Exception as e:
print(f"Error closing Redis connection: {e}")
print(f"分析段落时出错: {str(e)}")
continue
return analysis_results
async def merge_analysis_results(results: List[dict]) -> dict:
"""合并多个分析结果"""
if not results:
return {}
print(f"开始合并 {len(results)} 个分析结果...")
system_prompt = """
You are an AI assistant tasked with merging multiple analysis results of different parts of the same academic literature.
Generate a comprehensive merged analysis in JSON format.
The JSON structure must strictly follow the provided template.
Ensure the merged result is coherent and eliminates redundancy.
"""
user_prompt = f"""Merge the following analysis results into a single coherent analysis:
Analysis Results: {json.dumps(results, ensure_ascii=False)}
Generate a JSON response with the following structure:
{{
"1. Basic Information": {{
"author": "[Merged author information]",
"publication_date": "[Publication date]",
"title": "[Complete title]",
"journal_publisher": "[Journal/publisher information]",
"document_type": "[Document type]"
}},
"2. Content Analysis": {{
"abstract": "[Complete abstract]",
"research_purpose": "[Comprehensive research objectives]",
"methodology": "[Complete methodology description]",
"main_arguments": "[Comprehensive theoretical frameworks and arguments]",
"conclusions": "[Complete findings and conclusions]",
"innovations": "[Complete list of innovations]"
}}
}}
"""
result = await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
if result:
print("分析结果合并完成")
else:
print("分析结果合并失败")
return result
async def analyze_reference_document_async(content: str):
"""分析文献的基本信息和内容"""
system_prompt = """
You are an AI assistant tasked with analyzing academic literature.
Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
The JSON structure must strictly follow the provided template.
"""
user_prompt = f"""Analyze the following literature and extract all relevant information:
Content: {content}
Generate a JSON response with the following structure:
{{
"1. Basic Information": {{
"author": "[Author name(s) and affiliations]",
"publication_date": "[Publication date in YYYY-MM format]",
"title": "[Full title of the document]",
"journal_publisher": "[Journal name or publisher details]",
"document_type": "[Type: journal article/book/conference paper etc.]"
}},
"2. Content Analysis": {{
"abstract": "[Paper abstract]",
"research_purpose": "[Main objectives and research questions]",
"methodology": "[Research methods, data collection and analysis approaches]",
"main_arguments": "[Key theoretical frameworks and arguments]",
"conclusions": "[Primary findings and conclusions]",
"innovations": "[Novel contributions and original aspects]"
}}
}}
"""
return await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
async def analyze_reference_value_async(content_analysis: dict):
"""基于内容分析结果评估文献的价值"""
system_prompt = """
You are an AI assistant tasked with evaluating the value of academic literature based on its content analysis.
Generate a comprehensive value evaluation in JSON format.
The JSON structure must strictly follow the provided template.
"""
user_prompt = f"""Based on the following content analysis, evaluate the literature's value:
Content Analysis: {json.dumps(content_analysis, ensure_ascii=False)}
Generate a JSON response with the following structure:
{{
"3. Value Evaluation": {{
"academic_contribution": "[Significance to the field of study]",
"practical_significance": "[Real-world applications and implications]",
"limitations": "[Research constraints and weaknesses]",
"implications": "[Suggestions for future research and practice]"
}}
}}
"""
return await call_deepseek_api_async([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
])
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=6000)
uvicorn.run(app, host="0.0.0.0", port=6000)