Merge 6bfede2657 into d7a303b410

arxiv_daemon.py

@@ -12,6 +12,7 @@ import argparse
 
 from aslite.arxiv import get_response, parse_response
 from aslite.db import get_papers_db, get_metas_db
+from aslite.paperswithcode import getGithubLink
 
 if __name__ == '__main__':
 
@@ -68,7 +69,24 @@ if __name__ == '__main__':
         nhad, nnew, nreplace = 0, 0, 0
         for p in papers:
             pid = p['_id']
+            # add github_links to p
+            p['github_links'] = getGithubLink(pid)
+            # get github link from paperswithcode.com
             if pid in pdb:
+                # check if old papers have github_links
+                if 'github_links' in pdb[pid]:
+                    # replace if github has been updated by paperswithcode.com 
+                    if p['github_links'] != pdb[pid]['github_links']:
+                        # replace, this one is newer
+                        store(p)
+                        nreplace += 1
+                        continue
+                else:
+                    # replace, this one is newer
+                    store(p)
+                    nreplace += 1
+                    continue
+
                 if p['_time'] > pdb[pid]['_time']:
                     # replace, this one is newer
                     store(p)

aslite/paperswithcode.py

@@ -0,0 +1,23 @@
+import requests
+
+def getGithubLink(arxiv_id):
+    urls = []
+    base_url = "https://paperswithcode.com/api/v1/"
+    add_paper_id = "papers?arxiv_id=%s" %(arxiv_id)
+    search_query = base_url + add_paper_id
+    request = requests.get(search_query)
+    if request:
+        for _data in request.json()["results"]:
+            _id = _data['id']
+            add_id = "papers/%s/repositories/" %(_id)
+            search_url = base_url + add_id
+            request = requests.get(search_url)
+            if request:
+                for _urlInfo in request.json()["results"]:
+                    if _urlInfo['is_official']:
+                        urls.append(_urlInfo['url'])
+            else:
+                return None
+    else:
+        return None
+    return urls

bash_job.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+python3 arxiv_daemon.py --num 100
+
+if [ $? -eq 0 ]; then
+    echo "New papers detected! Running compute.py"
+    python3 compute.py
+else
+    echo "No new papers were added, skipping feature computation"
+fi

serve.py

@@ -100,6 +100,7 @@ def render_pid(pid):
         utags = [t for t, pids in tags.items() if pid in pids],
         summary = d['summary'],
         thumb_url = thumb_url,
+        github_links = d['github_links'],
     )
 
 def random_rank():

static/paper_list.js

@@ -12,6 +12,14 @@ const UTag = props => {
     )
 }
 
+const GithubLinks = props => {
+    const link = props.link;
+    return (<div class='rel_github_link'>
+        <a href={link}>Link</a>
+    </div>
+    )
+}
+
 const Paper = props => {
     const p = props.paper;
 
@@ -23,6 +31,12 @@ const Paper = props => {
     const similar_url = "/?rank=pid&pid=" + p.id;
     const inspect_url = "/inspect?pid=" + p.id;
     const thumb_img = p.thumb_url === '' ? null : <div class='rel_img'><img src={p.thumb_url} /></div>;
+    let links = null;
+    const github = p.github_links.length > 0 ? <div class='rel_github_link'>Github Links:</div> : null
+    if (p.github_links.length > 0){
+        links = p.github_links.map((link, ix) => <GithubLinks key={ix} link={link}/>)
+    }
+
     // if the user is logged in then we can show add/sub buttons
     let utag_controls = null;
     if(user) {
@@ -47,6 +61,8 @@ const Paper = props => {
         <div class='rel_abs'>{p.summary}</div>
         <div class='rel_more'><a href={similar_url}>similar</a></div>
         <div class='rel_inspect'><a href={inspect_url}>inspect</a></div>
+        {github}
+        {links}
     </div>
     )
 }

static/style.css

@@ -170,6 +170,11 @@ body {
     display: inline-block;
     margin-right: 10px;
 }
+.rel_github_link {
+    font-size: 14px;
+    display: inline-block;
+    margin-right: 10px;
+}
 #wordList {
     margin-top: 5px;
     margin-bottom: 10px;