obscura/zydi-paper
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新上传逻辑

Browse Source
This commit is contained in:
zydi
2025-01-19 06:05:40 +00:00
parent 1f256b1354
commit 339af355f9
2 changed files with 154 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
paper.html
+129 -229
View File
@@ -449,7 +449,7 @@
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span>Analyzing Paper content...</span>
<span>Checking analysis status...</span>
</div>
</div>
<div class="btn-group">
@@ -473,7 +473,7 @@
</div>
<div class="card-footer text-muted">
Upload Time: ${uploadTime}
<span class="ms-2 analysis-status-${paper.file_hash}">Analysis in progress...</span>
<span class="ms-2 analysis-status-${paper.file_hash}">Checking status...</span>
</div>
</div>
`;
@@ -508,22 +508,18 @@
papersList.innerHTML = paperCards;
// Get analysis status and results for each paper
papers.forEach(async (ref) => {
try {
const checkAnalysisStatus = async () => {
const analysisResponse = await fetch(`${API_BASE_URL}/report/${ref.file_hash}`);
if (!analysisResponse.ok) {
throw new Error('Failed to fetch analysis');
}
const analysisResult = await analysisResponse.json();
const checkPromises = papers.map(ref =>
fetch(`${API_BASE_URL}/report/${ref.file_hash}`)
.then(response => response.json())
.then(analysisResult => {
// 存储报告数据
paperReports.set(ref.file_hash, analysisResult);
const contentElement = document.querySelector(`.analysis-content-${ref.file_hash}`);
const statusElement = document.querySelector(`.analysis-status-${ref.file_hash}`);
if (!contentElement || !statusElement) return;
if (analysisResult.status === 'processing') {
// Processing status
contentElement.innerHTML = `
@@ -537,7 +533,7 @@
statusElement.innerHTML = '<span class="text-primary">Analysis in progress...</span>';
// 如果还在处理中，5秒后再次检查
setTimeout(checkAnalysisStatus, 10000);
setTimeout(() => checkAnalysisStatus(ref.file_hash), 10000);
} else if (analysisResult.status === 'completed') {
// Completed status
let author = 'N/A';
@@ -566,39 +562,44 @@
statusElement.innerHTML = '<span class="text-success">Analysis completed</span>';
} catch (parseError) {
console.error('Failed to parse analysis result:', parseError);
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Failed to parse analysis result
</div>
`;
statusElement.innerHTML = '<span class="text-danger">Failed to parse result</span>';
}
} else if (analysisResult.status === 'failed') {
// Failed status
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Analysis failed
Analysis failed, please upload again
</div>
<button class="btn btn-primary btn-sm" onclick="retryAnalysis('${ref.file_hash}', '${ref.paper_title}')">
<i class="bi bi-arrow-clockwise me-1"></i>
Retry
</button>
`;
statusElement.innerHTML = `<span class="text-danger">Analysis failed: ${analysisResult.message || 'Unknown error'}</span>`;
}
};
})
.catch(error => {
console.error(`Failed to fetch analysis for paper ${ref.file_hash}:`, error);
const contentElement = document.querySelector(`.analysis-content-${ref.file_hash}`);
const statusElement = document.querySelector(`.analysis-status-${ref.file_hash}`);
if (contentElement && statusElement) {
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Failed to get analysis status
</div>
`;
statusElement.innerHTML = '<span class="text-danger">Failed to get analysis status</span>';
}
})
);
// 开始首次检查
await checkAnalysisStatus();
} catch (error) {
console.error(`Failed to fetch analysis for paper ${ref.file_hash}:`, error);
const contentElement = document.querySelector(`.analysis-content-${ref.file_hash}`);
const statusElement = document.querySelector(`.analysis-status-${ref.file_hash}`);
contentElement.innerHTML = `
<div class="text-danger">
<i class="bi bi-exclamation-triangle me-2"></i>
Failed to get analysis status
</div>
`;
statusElement.innerHTML = '<span class="text-danger">Failed to get analysis status</span>';
}
});
// 等待所有状态检查完成
await Promise.all(checkPromises);
} catch (error) {
console.error('Failed to load Paper list:', error);
@@ -606,13 +607,42 @@
<div class="alert alert-danger">
Failed to load Paper list: ${error.message}
<br>
<button class="btn btn-primary mt-2" onclick="loadpapers()">
<button class="btn btn-primary mt-2" onclick="loadPapers()">
Retry
</button>
</div>`;
}
}
// Add check analysis status function
async function checkAnalysisStatus(fileHash) {
try {
const response = await fetch(`${API_BASE_URL}/report/${fileHash}`);
if (!response.ok) {
throw new Error('Failed to fetch analysis status');
}
const analysisResult = await response.json();
const contentElement = document.querySelector(`.analysis-content-${fileHash}`);
const statusElement = document.querySelector(`.analysis-status-${fileHash}`);
if (!contentElement || !statusElement) return;
// 更新报告数据
paperReports.set(fileHash, analysisResult);
if (analysisResult.status === 'processing') {
// 如果还在处理中，继续轮询
setTimeout(() => checkAnalysisStatus(fileHash), 10000);
} else {
// 重新加载论文列表以更新显示
await loadPapers();
}
} catch (error) {
console.error(`Failed to check analysis status for ${fileHash}:`, error);
}
}
async function uploadPaper() {
const input = document.createElement('input');
input.type = 'file';
@@ -676,102 +706,88 @@
const hashArray = Array.from(new Uint8Array(hashBuffer));
const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
// 检查是否已有报告
// 1. 首先检查是否已有分析报告
try {
const reportResponse = await fetch(`${API_BASE_URL}/check/${hashHex}`);
if (reportResponse.ok) {
const reportData = await reportResponse.json();
if (reportData.status === 'completed') {
progressText.textContent = `Report already exists for ${file.name}, skipping analysis...`;
progressText.textContent = `Report already exists for ${file.name}, skipping...`;
progressFill.style.width = '100%';
continue;
}
}
} catch (error) {
console.error('Error checking report:', error);
// 如果检查失败，继续上传流程
}
// 检查哈希值是否存在
// 2. 检查文件是否已存在于文件管理系统
let fileHash = hashHex;
try {
const checkResponse = await fetch(`https://files.aiot.ml/checkhash/${hashHex}`);
if (checkResponse.ok) {
// 哈希值已存在，直接使用
progressText.textContent = `File ${file.name} already exists, skipping upload...`;
progressFill.style.width = '100%';
const checkResult = await checkResponse.json();
if (!checkResult.exists) {
// 文件不存在,需要上传
progressText.textContent = `Uploading ${file.name}...`;
// 将文件信息发送到后端 API
const response = await fetch(`${API_BASE_URL}/upload`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
files: [{
filename: file.name,
hash: hashHex
}]
})
const formData = new FormData();
formData.append('file', file);
formData.append('client_hash', hashHex);
const xhr = new XMLHttpRequest();
xhr.upload.onprogress = function(e) {
if (e.lengthComputable) {
const percentComplete = (e.loaded / e.total) * 100;
progressFill.style.width = percentComplete + '%';
progressText.textContent = `Uploading ${file.name}: ${Math.round(percentComplete)}%`;
}
};
const uploadPromise = new Promise((resolve, reject) => {
xhr.onload = function() {
if (xhr.status === 200) {
resolve(JSON.parse(xhr.response));
} else {
reject(new Error('Upload failed'));
}
};
xhr.onerror = () => reject(new Error('Upload failed'));
});
if (!response.ok) {
throw new Error('Failed to process file');
}
continue;
xhr.open('POST', 'https://files.aiot.ml/upload');
xhr.send(formData);
const uploadResult = await uploadPromise;
fileHash = uploadResult.hash;
}
// 3. 调用分析接口
progressText.textContent = `Starting analysis for ${file.name}...`;
const response = await fetch(`${API_BASE_URL}/upload`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
files: [{
filename: file.name,
hash: fileHash
}]
})
});
if (!response.ok) {
throw new Error('Failed to process file');
}
progressText.textContent = `Analysis started for ${file.name}`;
progressFill.style.width = '100%';
} catch (error) {
console.error('Error checking hash:', error);
// 如果检查失败，继续上传流程
}
// 创建 FormData 对象
const formData = new FormData();
formData.append('file', file);
formData.append('client_hash', hashHex);
// 使用 XMLHttpRequest 来获取上传进度
const xhr = new XMLHttpRequest();
xhr.upload.onprogress = function(e) {
if (e.lengthComputable) {
const percentComplete = (e.loaded / e.total) * 100;
progressFill.style.width = percentComplete + '%';
progressText.textContent = `Uploading ${file.name}: ${Math.round(percentComplete)}%`;
}
};
const uploadPromise = new Promise((resolve, reject) => {
xhr.onload = function() {
if (xhr.status === 200) {
resolve(JSON.parse(xhr.response));
} else {
reject(new Error('Upload failed'));
}
};
xhr.onerror = () => reject(new Error('Upload failed'));
});
// 发送到文件存储服务
xhr.open('POST', 'https://files.aiot.ml/upload');
xhr.send(formData);
const uploadResult = await uploadPromise;
// 将文件信息发送到后端 API
const response = await fetch(`${API_BASE_URL}/upload`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
files: [{
filename: file.name,
hash: uploadResult.hash
}]
})
});
if (!response.ok) {
throw new Error('Failed to process file');
console.error('Error processing file:', error);
progressText.textContent = `Error processing ${file.name}: ${error.message}`;
continue;
}
}
@@ -1304,122 +1320,6 @@
alert('Failed to download report');
}
}
// 简化的重试函数，复用现有的上传流程
async function retryAnalysis(fileHash, paperTitle) {
try {
// 更新UI状态
const contentElement = document.querySelector(`.analysis-content-${fileHash}`);
const statusElement = document.querySelector(`.analysis-status-${fileHash}`);
contentElement.innerHTML = `
<div class="d-flex align-items-center">
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span>Retrying analysis...</span>
</div>
`;
statusElement.innerHTML = '<span class="text-primary">Retrying analysis...</span>';
// 检查文件是否存在
let fileExists = false;
try {
const checkResponse = await fetch(`https://files.aiot.ml/checkhash/${fileHash}`);
fileExists = checkResponse.ok;
} catch (error) {
console.error('Error checking hash:', error);
}
if (!fileExists) {
// 如果文件不存在，需要重新上传
try {
// 从原始URL获取文件内容
const fileResponse = await fetch(`https://files.aiot.ml/pdf/${fileHash}`);
if (!fileResponse.ok) {
throw new Error('Failed to fetch original file');
}
const fileBlob = await fileResponse.blob();
// 创建FormData对象重新上传
const formData = new FormData();
formData.append('file', fileBlob, paperTitle);
formData.append('client_hash', fileHash);
// 使用XMLHttpRequest上传文件
const xhr = new XMLHttpRequest();
// 创建上传进度提示
contentElement.innerHTML = `
<div class="d-flex align-items-center">
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span>Re-uploading file...</span>
</div>
`;
const uploadPromise = new Promise((resolve, reject) => {
xhr.onload = function() {
if (xhr.status === 200) {
resolve(JSON.parse(xhr.response));
} else {
reject(new Error('Upload failed'));
}
};
xhr.onerror = () => reject(new Error('Upload failed'));
});
// 发送到文件存储服务
xhr.open('POST', 'https://files.aiot.ml/upload');
xhr.send(formData);
await uploadPromise;
} catch (error) {
throw new Error(`Failed to re-upload file: ${error.message}`);
}
}
// 文件确认存在后，触发分析
const response = await fetch(`${API_BASE_URL}/upload`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
files: [{
filename: paperTitle,
hash: fileHash
}]
})
});
if (!response.ok) {
throw new Error('Failed to process file');
}
// 等待一段时间后刷新列表
setTimeout(async () => {
await loadPapers();
}, 2000);
} catch (error) {
console.error('Retry analysis failed:', error);
const contentElement = document.querySelector(`.analysis-content-${fileHash}`);
const statusElement = document.querySelector(`.analysis-status-${fileHash}`);
contentElement.innerHTML = `
<div class="text-danger mb-2">
<i class="bi bi-exclamation-triangle me-2"></i>
Retry failed
</div>
<button class="btn btn-primary btn-sm" onclick="retryAnalysis('${fileHash}', '${paperTitle}')">
<i class="bi bi-arrow-clockwise me-1"></i>
Retry Analysis
</button>
`;
statusElement.innerHTML = `<span class="text-danger">Retry failed: ${error.message}</span>`;
}
}
// 在页面加载完成后自动显示文献列表
document.addEventListener('DOMContentLoaded', function() {
showPaperList();
paper.py
+25 -47
View File
@@ -100,14 +100,14 @@ async def delete_paper(
try:
# 获取文献信息
paper = await db.papers.find_one({"_id": ObjectId(paper_id)})
paper = await db.papers.find_one({"file_hash": paper_id})
if not paper:
raise HTTPException(status_code=404, detail="paper not found")
# 从文件存储服务删除文件
file_hash = paper.get("file_hash")
# 删除数据库记录
await db.papers.delete_one({"_id": ObjectId(paper_id)})
await db.papers.delete_one({"file_hash": paper_id})
# 删除Redis中的分析报告（如果存在）
try:
@@ -193,9 +193,7 @@ class BatchUploadRequest(BaseModel):
files: List[FileUpload]
@app.post("/paper/upload")
async def batch_upload(
request: BatchUploadRequest
):
async def batch_upload(request: BatchUploadRequest):
"""批量上传项目相关文献"""
db = await get_database()
redis = await get_redis()
@@ -204,39 +202,11 @@ async def batch_upload(
uploaded_papers = []
papers_to_analyze = []
# 批量处理文件
for file in request.files:
try:
# 检查文件是否已存在分析报告
await redis.select(190)
report_key = f"paper_report:{file.hash}"
existing_report = await redis.get(report_key)
print(f"[Redis] existing_report: {existing_report}")
print(f"[Redis] file.hash: {file.hash}")
if existing_report:
report_data = json.loads(existing_report)
if report_data.get("status") == "completed":
print(f"[Redis] Report already exists for file {file.filename} with hash {file.hash}")
# 即使报告存在，也创建新的文献记录
paper = {
"paper_link": f"https://files.aiot.ml/pdf/content/{file.hash}",
"paper_title": file.filename,
"upload_time": datetime.now(timezone.utc),
"file_hash": file.hash
}
result = await db.papers.insert_one(paper)
paper_info = {
"paper_id": str(result.inserted_id),
"file_hash": file.hash,
"paper_title": paper["paper_title"]
}
uploaded_papers.append(paper_info)
continue
# 创建记录
# 1. 创建新记录
paper = {
"paper_link": f"https://files.aiot.ml/pdf/content/{file.hash}",
"paper_link": f"https://files.aiot.ml/pdf/{file.hash}",
"paper_title": file.filename,
"upload_time": datetime.now(timezone.utc),
"file_hash": file.hash
@@ -250,25 +220,33 @@ async def batch_upload(
}
uploaded_papers.append(paper_info)
# 如果没有现有报告或报告未完成，创建初始状态并添加到待分析列表
if not existing_report or json.loads(existing_report).get("status") != "completed":
initial_status = {
"status": "processing",
"message": "Analysis in progress"
}
await redis.set(report_key, json.dumps(initial_status))
papers_to_analyze.append(paper_info)
# 2. 设置分析状态
await redis.select(190)
report_key = f"paper_report:{file.hash}"
initial_status = {
"status": "processing",
"message": "Analysis in progress"
}
await redis.set(report_key, json.dumps(initial_status))
papers_to_analyze.append(paper_info)
except Exception as file_error:
print(f"处理文件 {file.filename} 时出错: {str(file_error)}")
uploaded_papers.append({
"file_hash": file.hash,
"paper_title": file.filename,
"status": "error",
"error_message": str(file_error)
})
continue
# 只对没有报告的文件启动分析任务
# 启动分析任务
if papers_to_analyze:
asyncio.create_task(batch_analysis(papers_to_analyze))
print(f"[Redis] 报告不存在，开始分析: {papers_to_analyze}")
print(f"[Redis] 开始分析新文件: {papers_to_analyze}")
return {
"message": f"Successfully uploaded {len(uploaded_papers)} files",
"message": f"Successfully processed {len(uploaded_papers)} files",
"uploaded_files": uploaded_papers,
"files_to_analyze": len(papers_to_analyze)
}
@@ -307,7 +285,7 @@ async def batch_analysis(papers: List[dict]):
return
# 从文件存储服务获取PDF内容
async with aiohttp.ClientSession() as session:
async with session.get(f'https://files.aiot.ml/pdf/content/{file_hash}') as response:
async with session.get(f'https://files.aiot.ml/pdf/{file_hash}') as response:
if response.status != 200:
print(f"[Redis] 获取PDF内容失败: {response.status}")
raise Exception(f"Failed to get PDF content for file hash: {file_hash}")