Add dates and mailing list names to search results

change "documents" to "issues" in reindexing message

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,17 @@
+Thanks for using Centillion. Your feedback is important to us. 
+
+### When reporting a bug, please be sure to include the following:
+- [ ] A descriptive title
+- [ ] The behavior you expect to see and the actual behavior observed
+- [ ] Steps to reproduce the behavior 
+- [ ] What browser you are using
+
+### When you open an issue for a feature request, please add as much detail as possible:
+- [ ] A descriptive title
+- [ ] A description of the problem you're trying to solve, including *why* you think this is a problem
+- [ ] An overview of the suggested solution
+- [ ] If the feature changes current behavior, please explain why your solution is better
+
+See read [our contributor guidelines](https://github.com/dcppc/centillion/blob/dcppc/CONTRIBUTING.md) 
+for more details about contributing to this project.
+

centillion_search.py

@@ -744,10 +744,10 @@ class Search:
                 full_items[f['id']] = f
             
             ## Shorter:
-            break
-            ## Longer:
-            #if nextPageToken is None:
             #break
+            ## Longer:
+            if nextPageToken is None:
+                break
 
 
         writer = self.ix.writer()
@@ -795,7 +795,7 @@ class Search:
         subprocess.call(['rm','-fr',temp_dir])
 
         writer.commit()
-        print("Done, updated %d documents in the index" % count)
+        print("Done, updated %d Google Drive files in the index" % count)
 
 
     # ------------------------------
@@ -873,7 +873,7 @@ class Search:
 
 
         writer.commit()
-        print("Done, updated %d documents in the index" % count)
+        print("Done, updated %d Github issues in the index" % count)
 
 
 
@@ -1165,9 +1165,9 @@ class Search:
             sr.id = r['id']
             sr.kind = r['kind']
 
-            sr.created_time = r['created_time']
+            sr.created_time =  datetime.strftime(r['created_time'],  "%Y-%m-%d %I:%M %p")
-            sr.modified_time = r['modified_time']
+            sr.modified_time = datetime.strftime(r['modified_time'], "%Y-%m-%d %I:%M %p")
-            sr.indexed_time = r['indexed_time']
+            sr.indexed_time =  datetime.strftime(r['indexed_time'],  "%Y-%m-%d %I:%M %p")
 
             sr.title = r['title']
             sr.url = r['url']
@@ -1177,6 +1177,8 @@ class Search:
             sr.owner_email = r['owner_email']
             sr.owner_name = r['owner_name']
 
+            sr.group = r['group']
+
             sr.repo_name = r['repo_name']
             sr.repo_url = r['repo_url']
 

config_flask.example.py

@@ -1,20 +1,38 @@
+######################################
+# github oauth
+GITHUB_OAUTH_CLIENT_ID = "XXX"
+GITHUB_OAUTH_CLIENT_SECRET = "YYY"
+
+######################################
+# github acces token
+GITHUB_TOKEN = "XXX"
+
+######################################
+# groups.io
+GROUPSIO_TOKEN = "XXXXX"
+GROUPSIO_USERNAME = "XXXXX"
+GROUPSIO_PASSWORD = "XXXXX"
+
+######################################
+# Disqus API public key
+DISQUS_TOKEN = "XXXXX"
+
+######################################
+# everything else
+
 # Location of index file
 INDEX_DIR = "search_index"
 
-# oauth client deets
-GITHUB_OAUTH_CLIENT_ID = "XXX"
-GITHUB_OAUTH_CLIENT_SECRET = "YYY"
-GITHUB_TOKEN = "ZZZ"
-
 # More information footer: Repository label
-FOOTER_REPO_ORG = "charlesreid1"
+FOOTER_REPO_ORG = "dcppc"
 FOOTER_REPO_NAME = "centillion"
 
 # Toggle to show Whoosh parsed query
 SHOW_PARSED_QUERY=True
 
-TAGLINE = "Search All The Things"
+TAGLINE = "Search the Data Commons"
 
 # Flask settings
 DEBUG = True
-SECRET_KEY = 'WWWWW'
+SECRET_KEY = 'XXXXX'
+

disqus_util.py

@@ -118,6 +118,7 @@ class DisqusCrawler(object):
 
                         link = response['link']
                         clean_link = re.sub('data-commons.us','nihdatacommons.us',link)
+                        clean_link += "#disqus_comments"
 
                         # Finished working on thread.
 

groupsio_util.py

@@ -66,7 +66,7 @@ class GroupsIOArchivesCrawler(object):
 
             ## Short circuit
             ## for debugging purposes
-            break
+            #break
 
         return subgroups
 

static/style.css

@@ -58,7 +58,7 @@ button#feedback {
 /* search results table */
 td#search-results-score-col,
 td#search-results-type-col {
-    width: 100px;
+    width: 90px;
 }
 
 div.container {
@@ -86,6 +86,14 @@ div.container {
 }
 
 /* badges for number of docs indexed */
+span.results-count {
+    background-color: #555;
+}
+
+span.indexing-count {
+    background-color: #337ab7;
+}
+
 span.badge {
     vertical-align: text-bottom;
 }
@@ -126,7 +134,7 @@ li.search-group-item {
 }
 
 div.url {
-    background-color: rgba(86,61,124,.15);
+    background-color: rgba(40,40,60,.15);
     padding: 8px;
 }
 
@@ -192,7 +200,7 @@ table {
 
 .info, .last-searches {
     color: gray;
-    font-size: 12px;
+    /*font-size: 12px;*/
     font-family: Arial, serif;
 }
 
@@ -202,27 +210,27 @@ table {
 
 div.tags a, td.tag-cloud a {
     color: #b56020;
-    font-size: 12px;
+    /*font-size: 12px;*/
 }
 
 td.tag-cloud, td.directories-cloud {
-    font-size: 12px;
+    /*font-size: 12px;*/
     color: #555555;
 }
 
 td.directories-cloud a {
-    font-size: 12px;
+    /*font-size: 12px;*/
     color: #377BA8;
 }
 
 div.path {
-    font-size: 12px;
+    /*font-size: 12px;*/
     color: #666666;
     margin-bottom: 3px;
 }
 
 div.path a {
-    font-size: 12px;
+    /*font-size: 12px;*/
     margin-right: 5px;
 }
 

templates/search.html

@@ -52,8 +52,8 @@
                     <div class="container-fluid">
                         <div class="row">
                             <div class="col-xs-12 info">
-                                <b>Found:</b> <span class="badge">{{entries|length}}</span> results 
+                                <b>Found:</b> <span class="badge results-count">{{entries|length}}</span> results 
-                                out of <span class="badge">{{totals["total"]}}</span> total items indexed
+                                out of <span class="badge results-count">{{totals["total"]}}</span> total items indexed
                             </div>
                         </div>
                     </div>
@@ -67,32 +67,32 @@
                             <div class="col-xs-12 info">
                                 <b>Indexing:</b>
 
-                                <span class="badge">{{totals["gdoc"]}}</span>
+                                <span class="badge indexing-count">{{totals["gdoc"]}}</span>
                                 <a href="/master_list?doctype=gdoc#gdoc">
                                 Google Drive files
                                 </a>,
 
-                                <span class="badge">{{totals["issue"]}}</span>
+                                <span class="badge indexing-count">{{totals["issue"]}}</span>
                                 <a href="/master_list?doctype=issue#issue">
                                 Github issues
                                 </a>,
 
-                                <span class="badge">{{totals["ghfile"]}}</span>
+                                <span class="badge indexing-count">{{totals["ghfile"]}}</span>
                                 <a href="/master_list?doctype=ghfile#ghfile">
                                 Github files
                                 </a>,
 
-                                <span class="badge">{{totals["markdown"]}}</span>
+                                <span class="badge indexing-count">{{totals["markdown"]}}</span>
                                 <a href="/master_list?doctype=markdown#markdown">
                                 Github Markdown files
                                 </a>,
 
-                                <span class="badge">{{totals["emailthread"]}}</span>
+                                <span class="badge indexing-count">{{totals["emailthread"]}}</span>
                                 <a href="/master_list?doctype=emailthread#emailthread">
                                 Groups.io email threads
                                 </a>,
 
-                                <span class="badge">{{totals["disqus"]}}</span>
+                                <span class="badge indexing-count">{{totals["disqus"]}}</span>
                                 <a href="/master_list?doctype=disqus#disqus">
                                 Disqus comment threads
                                 </a>
@@ -101,6 +101,7 @@
                 </div>
             </li>
 
+
         </ul>
     </div>
 </div>
@@ -164,10 +165,18 @@
                                                 <a href='{{e.url}}'>{{e.title}}</a>
                                                 (Owner: {{e.owner_name}}, {{e.owner_email}})<br />
                                                 <b>Document Type</b>: {{e.mimetype}}
+                                                {% if e.created_time %} 
+                                                    <br/>
+                                                    <b>Created:</b> {{e.created_time}}
+                                                {% endif %}
                                             {% else %}
                                                 <b>Google Drive File:</b>
                                                 <a href='{{e.url}}'>{{e.title}}</a><br />
                                                 <b>Owner:</b> {{e.owner_name}}, {{e.owner_email}}<br />
+                                                {% if e.created_time %} 
+                                                    <br/>
+                                                    <b>Created:</b> {{e.created_time}}
+                                                {% endif %}
                                             {% endif %}
 
                                         {% elif e.kind=="issue" %}
@@ -178,6 +187,10 @@
                                             {% endif %}
                                             <br/>
                                             <b>Repository:</b> <a href='{{e.repo_url}}'>{{e.repo_name}}</a>
+                                            {% if e.created_time %} 
+                                                <br/>
+                                                <b>Date:</b> {{e.created_time}}
+                                            {% endif %}
 
                                         {% elif e.kind=="ghfile" %}
                                             <b>Github File:</b>
@@ -196,6 +209,21 @@
                                             <a href='{{e.url}}'>{{e.title}}</a>
                                             <br/>
                                             <b>Started By:</b> {{e.owner_name}}
+                                            <br/>
+                                            <b>Mailing List:</b> {{e.group}}
+                                            {% if e.created_time %} 
+                                                <br/>
+                                                <b>Date:</b> {{e.created_time}}
+                                            {% endif %}
+
+                                        {% elif e.kind=="disqus" %}
+                                            <b>Disqus Comment Thread:</b>
+                                            <a href='{{e.url}}'>{{e.title}}</a>
+                                            <br/>
+                                            {% if e.created_time %} 
+                                                <br/>
+                                                <b>Date:</b> {{e.created_time}}
+                                            {% endif %}
 
                                         {% else %}
                                             <b>Item:</b> (<a href='{{e.url}}'>link</a>)