diff --git a/literature.html b/literature.html
index 3fcb32c..36101be 100644
--- a/literature.html
+++ b/literature.html
@@ -654,70 +654,78 @@
// Get analysis status and results for each literature
references.forEach(async (ref) => {
try {
- const analysisResponse = await fetch(`${API_BASE_URL}/references/${ref._id}/report`, {
- headers: {
- 'Authorization': `Bearer ${token}`
- }
- });
-
- if (!analysisResponse.ok) {
- throw new Error('Failed to fetch analysis');
- }
-
- const analysisResult = await analysisResponse.json();
- const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
- const statusElement = document.querySelector(`.analysis-status-${ref._id}`);
-
- if (analysisResult.status === 'processing') {
- // Processing status
- contentElement.innerHTML = `
-
-
- Loading...
-
-
Analyzing literature content...
-
Author: ${author}
- Publication Date: ${publicationDate}
- Journal/Publisher: ${journal}
- `;
- statusElement.innerHTML = 'Analysis completed';
- } catch (parseError) {
- console.error('Failed to parse analysis result:', parseError);
+ });
+
+ if (!analysisResponse.ok) {
+ throw new Error('Failed to fetch analysis');
}
- } else if (analysisResult.status === 'failed') {
- // Failed status
- contentElement.innerHTML = `
-
-
- Analysis failed
-
- `;
- statusElement.innerHTML = `Analysis failed: ${analysisResult.message || 'Unknown error'}`;
- }
+
+ const analysisResult = await analysisResponse.json();
+ const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
+ const statusElement = document.querySelector(`.analysis-status-${ref._id}`);
+
+ if (analysisResult.status === 'processing') {
+ // Processing status
+ contentElement.innerHTML = `
+
+
+ Loading...
+
+
Analyzing literature content...
+
Author: ${author}
+ Publication Date: ${publicationDate}
+ Journal/Publisher: ${journal}
+ `;
+ statusElement.innerHTML = 'Analysis completed';
+ } catch (parseError) {
+ console.error('Failed to parse analysis result:', parseError);
+ }
+ } else if (analysisResult.status === 'failed') {
+ // Failed status
+ contentElement.innerHTML = `
+
+
+ Analysis failed
+
+ `;
+ statusElement.innerHTML = `Analysis failed: ${analysisResult.message || 'Unknown error'}`;
+ }
+ };
+
+ // 开始首次检查
+ await checkAnalysisStatus();
} catch (error) {
console.error(`Failed to fetch analysis for reference ${ref._id}:`, error);
const contentElement = document.querySelector(`.analysis-content-${ref._id}`);
diff --git a/main.py b/main.py
index b6894dd..19da826 100644
--- a/main.py
+++ b/main.py
@@ -1648,71 +1648,7 @@ async def analyze_project_data(project_data):
print(f"Error calling DeepSeek API: {e}")
return None
-async def analyze_reference_document_async(content: str):
- """分析文献的基本信息和内容"""
- system_prompt = """
- You are an AI assistant tasked with analyzing academic literature.
- Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
- The JSON structure must strictly follow the provided template.
- """
-
- user_prompt = f"""Analyze the following literature and extract all relevant information:
- Content: {content}
- Generate a JSON response with the following structure:
-
- {{
- "1. Basic Information": {{
- "author": "[Author name(s) and affiliations]",
- "publication_date": "[Publication date in YYYY-MM format]",
- "title": "[Full title of the document]",
- "journal_publisher": "[Journal name or publisher details]",
- "document_type": "[Type: journal article/book/conference paper etc.]"
- }},
- "2. Content Analysis": {{
- "abstract": "[Paper abstract]",
- "research_purpose": "[Main objectives and research questions]",
- "methodology": "[Research methods, data collection and analysis approaches]",
- "main_arguments": "[Key theoretical frameworks and arguments]",
- "conclusions": "[Primary findings and conclusions]",
- "innovations": "[Novel contributions and original aspects]"
- }}
- }}
- """
-
- return await call_deepseek_api_async([
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt}
- ])
-
-async def analyze_reference_value_async(content_analysis: dict):
- """基于内容分析结果评估文献的价值"""
- system_prompt = """
- You are an AI assistant tasked with evaluating the value of academic literature based on its content analysis.
- Generate a comprehensive value evaluation in JSON format.
- The JSON structure must strictly follow the provided template.
- """
-
- user_prompt = f"""Based on the following content analysis, evaluate the literature's value:
- Content Analysis: {json.dumps(content_analysis, ensure_ascii=False)}
-
- Generate a JSON response with the following structure:
-
- {{
- "3. Value Evaluation": {{
- "academic_contribution": "[Significance to the field of study]",
- "practical_significance": "[Real-world applications and implications]",
- "limitations": "[Research constraints and weaknesses]",
- "implications": "[Suggestions for future research and practice]"
- }}
- }}
- """
-
- return await call_deepseek_api_async([
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt}
- ])
-
# 修改函数定义为异步函数
async def analyze_reference_summary(reference_data):
system_prompt = """
@@ -2810,25 +2746,72 @@ async def process_batch_analysis(references: List[dict]):
print(f"Reference file not found: {file_path}")
return
+ print(f"\n开始处理文献 {ref.get('reference_title', '未知标题')}")
+ print(f"文献ID: {reference_id}")
+ print(f"文件路径: {file_path}")
+
# 异步读取PDF
+ print("\n=== 步骤1: 读取PDF文件 ===")
pdf_content = await read_pdf_async(file_path)
if not pdf_content:
raise Exception("Failed to read PDF content")
- # 异步分析文档
- document_analysis = await analyze_reference_document_async(pdf_content)
- if not document_analysis:
- raise Exception("Failed to analyze document")
+ # 打印字符数
+ content_length = len(pdf_content)
+ print(f"\n=== 步骤2: 内容长度检查 ===")
+ print(f"PDF内容总字符数: {content_length}")
+
+ # 根据内容长度选择不同的处理方式
+ if content_length <= 200000:
+ print(f"\n=== 步骤3A: 使用直接分析方式 ===")
+ print(f"文档长度在处理范围内 ({content_length} <= 200000)")
+ # 直接分析文档内容
+ print("开始分析文档内容...")
+ document_analysis = await analyze_reference_document_async(pdf_content[:180000])
+ if not document_analysis:
+ raise Exception("Failed to analyze document")
+ print("文档分析完成")
+ else:
+ print(f"\n=== 步骤3B: 使用分段分析方式 ===")
+ print(f"文档超过200000字符 ({content_length} > 200000)")
+ # 分段分析长文档
+ print("\n--- 开始分段分析 ---")
+ print("正在调用 analyze_long_document_async...")
+ try:
+ analysis_results = await analyze_long_document_async(pdf_content)
+ except Exception as e:
+ print(f"分段分析过程中出错: {str(e)}")
+ raise
+
+ if not analysis_results:
+ raise Exception("Failed to analyze document in segments")
+ print(f"分段分析完成，共分析了 {len(analysis_results)} 个段落")
+
+ # 合并分析结果
+ try:
+ document_analysis = await merge_analysis_results(analysis_results)
+ except Exception as e:
+ raise
+
+ if not document_analysis:
+ raise Exception("Failed to merge analysis results")
# 等待一小段时间避免API限制
await asyncio.sleep(1)
- # 异步分析价值
- value_evaluation = await analyze_reference_value_async(document_analysis)
+ # 异步分析文献价值
+ try:
+ value_evaluation = await analyze_reference_value_async(document_analysis)
+ print(f"analyze_reference_value_async 返回结果类型: {type(value_evaluation)}")
+ except Exception as e:
+ raise
+
if not value_evaluation:
raise Exception("Failed to evaluate value")
+ print("文献价值分析完成")
# 合并结果
+ print("\n=== 步骤5: 保存最终结果 ===")
analysis_result = {
**document_analysis,
**value_evaluation,
@@ -2841,7 +2824,6 @@ async def process_batch_analysis(references: List[dict]):
await redis.set(report_key, json.dumps(analysis_result))
except Exception as e:
- print(f"Error processing reference {ref['reference_id']}: {str(e)}")
try:
await redis.select(203)
report_key = f"reference_report:{ref['reference_id']}"
@@ -2884,7 +2866,21 @@ async def read_pdf_async(file_path: str) -> str:
for page in pdf_reader.pages:
content = page.extract_text()
pdf_content += content
- return pdf_content[:180000] # 限制内容长度
+
+ # 根据内容长度返回不同的结果
+ content_length = len(pdf_content)
+ print(f"\nPDF原始内容字符数: {content_length}")
+
+ if content_length <= 180000:
+ print("文档长度 ≤ 180000，返回完整内容")
+ return pdf_content
+ elif content_length <= 200000:
+ print("文档长度在 180000-200000 之间，截取前 180000 个字符")
+ return pdf_content[:180000]
+ else:
+ print("文档长度 > 200000，返回完整内容供分段处理")
+ return pdf_content # 返回完整内容，由调用者处理分段
+
except Exception as e:
print(f"Error reading PDF: {e}")
return ""
@@ -2906,62 +2902,191 @@ async def call_deepseek_api_async(messages: list) -> dict:
else:
raise Exception(f"API调用失败: {await response.text()}")
-async def process_reference_analysis(reference_id: str, reference: dict):
- """后台处理文献分析"""
- redis = await get_redis()
+async def analyze_long_document_async(content: str) -> List[dict]:
+ """分段分析长文档"""
+ # 将内容分成多个段落，每段约60000个字符（估算后约50000 tokens）
+ segments = []
+ content_length = len(content)
+ segment_size = 60000 # 减小段落大小
- try:
- file_path = reference.get("reference_link")
- if not file_path or not os.path.exists(file_path):
- print(f"Reference file not found: {file_path}")
- return
-
- # 异步读取PDF
- pdf_content = await read_pdf_async(file_path)
- if not pdf_content:
- raise Exception("Failed to read PDF content")
+ print(f"\n开始分段处理，总字符数: {content_length}")
+ print(f"每段大小: {segment_size} 字符")
+
+ for i in range(0, content_length, segment_size):
+ segment = content[i:i + segment_size]
+ segments.append(segment)
+
+ print(f"文档已分段，共 {len(segments)} 个段落")
+
+ # 对每个段落进行分析
+ analysis_results = []
+ for i, segment in enumerate(segments):
+ print(f"\n开始分析第 {i+1}/{len(segments)} 个段落...")
- # 异步分析文档内容
- document_analysis = await analyze_reference_document_async(pdf_content)
- if not document_analysis:
- print("Failed to analyze document")
- return
-
- # 异步分析文献价值
- value_evaluation = await analyze_reference_value_async(document_analysis)
- if not value_evaluation:
- print("Failed to evaluate value")
- return
-
- # 合并分析结果
- analysis_result = {
- **document_analysis,
- **value_evaluation
- }
+ system_prompt = f"""
+ You are an AI assistant tasked with analyzing part {i+1} of {len(segments)} of an academic literature.
+ Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
+ Note that this is part {i+1} of a longer document, so focus on the content provided.
+ The JSON structure must strictly follow the provided template.
+ """
+
+ user_prompt = f"""Analyze the following literature segment and extract all relevant information:
+ Content: {segment}
- # 保存分析结果到db203
- await redis.select(203)
- report_key = f"reference_report:{reference_id}"
- await redis.set(report_key, json.dumps(analysis_result))
+ Generate a JSON response with the following structure:
+
+ {{
+ "1. Basic Information": {{
+ "author": "[Author name(s) and affiliations]",
+ "publication_date": "[Publication date in YYYY-MM format]",
+ "title": "[Full title of the document]",
+ "journal_publisher": "[Journal name or publisher details]",
+ "document_type": "[Type: journal article/book/conference paper etc.]"
+ }},
+ "2. Content Analysis": {{
+ "abstract": "[Paper abstract]",
+ "research_purpose": "[Main objectives and research questions]",
+ "methodology": "[Research methods, data collection and analysis approaches]",
+ "main_arguments": "[Key theoretical frameworks and arguments]",
+ "conclusions": "[Primary findings and conclusions]",
+ "innovations": "[Novel contributions and original aspects]"
+ }}
+ }}
+ """
- except Exception as e:
- print(f"Error analyzing reference data: {str(e)}")
try:
- await redis.select(203)
- report_key = f"reference_report:{reference_id}"
- error_status = {
- "status": "failed",
- "message": str(e)
- }
- await redis.set(report_key, json.dumps(error_status))
- except Exception as redis_error:
- print(f"Error updating Redis status: {redis_error}")
- finally:
- try:
- await redis.aclose()
+ result = await call_deepseek_api_async([
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ])
+
+ if result:
+ print(f"第 {i+1} 个段落分析完成")
+ analysis_results.append(result)
+ # 等待一小段时间避免API限制
+ await asyncio.sleep(1)
+ else:
+ print(f"第 {i+1} 个段落分析失败")
except Exception as e:
- print(f"Error closing Redis connection: {e}")
+ print(f"分析段落时出错: {str(e)}")
+ continue
+
+ return analysis_results
+async def merge_analysis_results(results: List[dict]) -> dict:
+ """合并多个分析结果"""
+ if not results:
+ return {}
+
+ print(f"开始合并 {len(results)} 个分析结果...")
+
+ system_prompt = """
+ You are an AI assistant tasked with merging multiple analysis results of different parts of the same academic literature.
+ Generate a comprehensive merged analysis in JSON format.
+ The JSON structure must strictly follow the provided template.
+ Ensure the merged result is coherent and eliminates redundancy.
+ """
+
+ user_prompt = f"""Merge the following analysis results into a single coherent analysis:
+ Analysis Results: {json.dumps(results, ensure_ascii=False)}
+
+ Generate a JSON response with the following structure:
+
+ {{
+ "1. Basic Information": {{
+ "author": "[Merged author information]",
+ "publication_date": "[Publication date]",
+ "title": "[Complete title]",
+ "journal_publisher": "[Journal/publisher information]",
+ "document_type": "[Document type]"
+ }},
+ "2. Content Analysis": {{
+ "abstract": "[Complete abstract]",
+ "research_purpose": "[Comprehensive research objectives]",
+ "methodology": "[Complete methodology description]",
+ "main_arguments": "[Comprehensive theoretical frameworks and arguments]",
+ "conclusions": "[Complete findings and conclusions]",
+ "innovations": "[Complete list of innovations]"
+ }}
+ }}
+ """
+
+ result = await call_deepseek_api_async([
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ])
+
+ if result:
+ print("分析结果合并完成")
+ else:
+ print("分析结果合并失败")
+
+ return result
+async def analyze_reference_document_async(content: str):
+ """分析文献的基本信息和内容"""
+ system_prompt = """
+ You are an AI assistant tasked with analyzing academic literature.
+ Generate a comprehensive analysis in JSON format covering both basic information and content analysis.
+ The JSON structure must strictly follow the provided template.
+ """
+
+ user_prompt = f"""Analyze the following literature and extract all relevant information:
+ Content: {content}
+
+ Generate a JSON response with the following structure:
+
+ {{
+ "1. Basic Information": {{
+ "author": "[Author name(s) and affiliations]",
+ "publication_date": "[Publication date in YYYY-MM format]",
+ "title": "[Full title of the document]",
+ "journal_publisher": "[Journal name or publisher details]",
+ "document_type": "[Type: journal article/book/conference paper etc.]"
+ }},
+ "2. Content Analysis": {{
+ "abstract": "[Paper abstract]",
+ "research_purpose": "[Main objectives and research questions]",
+ "methodology": "[Research methods, data collection and analysis approaches]",
+ "main_arguments": "[Key theoretical frameworks and arguments]",
+ "conclusions": "[Primary findings and conclusions]",
+ "innovations": "[Novel contributions and original aspects]"
+ }}
+ }}
+ """
+
+ return await call_deepseek_api_async([
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ])
+
+async def analyze_reference_value_async(content_analysis: dict):
+ """基于内容分析结果评估文献的价值"""
+ system_prompt = """
+ You are an AI assistant tasked with evaluating the value of academic literature based on its content analysis.
+ Generate a comprehensive value evaluation in JSON format.
+ The JSON structure must strictly follow the provided template.
+ """
+
+ user_prompt = f"""Based on the following content analysis, evaluate the literature's value:
+ Content Analysis: {json.dumps(content_analysis, ensure_ascii=False)}
+
+ Generate a JSON response with the following structure:
+
+ {{
+ "3. Value Evaluation": {{
+ "academic_contribution": "[Significance to the field of study]",
+ "practical_significance": "[Real-world applications and implications]",
+ "limitations": "[Research constraints and weaknesses]",
+ "implications": "[Suggestions for future research and practice]"
+ }}
+ }}
+ """
+
+ return await call_deepseek_api_async([
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ])
+
if __name__ == "__main__":
import uvicorn
- uvicorn.run(app, host="0.0.0.0", port=6000)
\ No newline at end of file
+ uvicorn.run(app, host="0.0.0.0", port=6000)