Merge branch 'hotfix/search-box' into releases/v1

* hotfix/search-box:
  fix "Search Metadata" label so search does not break

.gitignore

@@ -1,4 +1,4 @@
-feedback_database.json
+config_centillion.py
 config_flask.py
 vp
 credentials.json

Readme.md

@@ -12,13 +12,8 @@ one centillion is 3.03 log-times better than a googol.
 ## What Is It
 
 Centillion (https://github.com/dcppc/centillion) is a search engine that can index 
-different kinds of document collections: Google Documents (.docx files), Google Drive files,
+three kinds of collections: Google Documents (.docx files), Github issues, and Markdown files in 
-Github issues, Github files, Github Markdown files, and Groups.io email threads.
+Github repos.
-
-
-
-
-## What Is It
 
 We define the types of documents the centillion should index,
 what info and how. The centillion then builds and

Todo.md

@@ -1,29 +1,47 @@
 # todo
 
-ux improvements:
+Main task:
-- feedback tools
+- hashing and caching
-- integrating master list into single list
+    - <s>first, working out the logic of how we group items into sets
-- providing advanced search interfce
+        - needs to be deleted
+        - needs to be updated
+        - needs to be added
+        - for docs, issues, and comments</s>
+    - second, when we add or update an item, need to:
+        - go through the motions, download file, extract text
+        - check for existing indexed doc with that id
+        - check if existing indexed doc has same hash
+            - if so, skip
+            - otherwise, delete and re-index
 
-big picture improvements:
+Other bugs:
-- hypothesis API
+- Some github issues have no title (?)
-- folksonomy tagging with hypothesis
+- <s>Need to combine issues with comments</s>
-- tags, expanded schema
+- Not able to index markdown files _in a repo_
+- (Longer term) update main index vs update diff index
+
+Needs:
+- <s>control panel</s>
+
+Thursday product:
+- Everything re-indexed nightly
+- Search engine built on all documents in Google Drive, all issues, markdown files
+- Using pandoc to extract Google Drive document contents
+- BRIEF quickstart documentation
+
+Future:
+- Future plans to improve - plugins, improving matching
+- Subdomain plans
+- Folksonomy tagging and integration plans
 
 
 
 
-feedback form: where we are at
+config options for plugins
-- feedback button
+conditional blocks with import github inside
-- button triggers modal form
+complicated tho - better to have components split off
-- modal has emojis for feedback, text box, buttons
+
-- clicking emojis changes color, to select
+
-- clicking submit with filled out form submits to an endpoint
-- clicking submit also closes form, but only if submit successful
-
-feedback form: what we need to do
-- fix alerts - thank you for your feedback doesn't show up until a refresh
-- probably an easy ajax fix
 
 
 

centillion.py

@@ -3,7 +3,6 @@ import subprocess
 
 import codecs
 import os, json
-from datetime import datetime
 
 from werkzeug.contrib.fixers import ProxyFix
 from flask import Flask, request, redirect, url_for, render_template, flash, jsonify
@@ -263,57 +262,13 @@ def list_docs(doctype):
         all_orgs = resp.json()
         for org in all_orgs:
             if org['login']=='dcppc':
-                # Business as usual
+                copper_team_id = '2700235'
-                search = Search(app.config["INDEX_DIR"])
+                mresp = github.get('/teams/%s/members/%s'%(copper_team_id,username))
-                return jsonify(search.get_list(doctype))
+                if mresp.status_code==204:
-
-    # nope
-    return render_template('403.html')
-
-
-@app.route('/feedback', methods=['POST'])
-def parse_request():
-
-    if not github.authorized:
-        return redirect(url_for("github.login"))
-    username = github.get("/user").json()['login']
-    resp = github.get("/user/orgs")
-    if resp.ok:
-        all_orgs = resp.json()
-        for org in all_orgs:
-            if org['login']=='dcppc':
-
-
-                try:
                     # Business as usual
-                    data = request.form.to_dict();
+                    search = Search(app.config["INDEX_DIR"])
-                    data['github_login'] = username
+                    return jsonify(search.get_list(doctype))
-                    data['timestamp'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
-                    feedback_database = 'feedback_database.json'
-                    if not os.path.isfile(feedback_database):
-                        with open(feedback_database,'w') as f:
-                            json_data = [data]
-                            json.dump(json_data, f, indent=4)
-
-                    else:
-                        json_data = []
-                        with open(feedback_database,'r') as f:
-                            json_data = json.load(f)
-
-                        json_data.append(data)
-
-                        with open(feedback_database,'w') as f:
-                            json.dump(json_data, f, indent=4)
-
-                    ## Should be done with Javascript
-                    #flash("Thank you for your feedback!")
-                    return jsonify({'status':'ok','message':'Thank you for your feedback!'})
-                except:
-                    return jsonify({'status':'error','message':'An error was encountered while submitting your feedback. Try submitting an issue in the <a href="https://github.com/dcppc/centillion/issues/new">dcppc/centillion</a> repository.'})
-
-
-    # nope
     return render_template('403.html')
 
 @app.errorhandler(404)
@@ -342,10 +297,5 @@ def store_search(query, fields):
 if __name__ == '__main__':
     # if running local instance, set to true
     os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = 'true'
-    port = os.environ.get('CENTILLION_PORT','')
+    app.run(host="0.0.0.0",port=5000)
-    if port=='':
-        port = 5000
-    else:
-        port = int(port)
-    app.run(host="0.0.0.0",port=port)
 

centillion_search.py

@@ -21,8 +21,6 @@ import dateutil.parser
 
 from whoosh.qparser import MultifieldParser, QueryParser
 from whoosh.analysis import StemmingAnalyzer
-from whoosh.qparser.dateparse import DateParserPlugin
-from whoosh import fields, index
 
 
 """
@@ -182,38 +180,30 @@ class Search:
         # is defined.
 
         schema = Schema(
-                id = fields.ID(stored=True, unique=True),
+                id = ID(stored=True, unique=True),
-                kind = fields.ID(stored=True),
+                kind = ID(stored=True),
 
-                created_time = fields.DATETIME(stored=True),
+                created_time = ID(stored=True),
-                modified_time = fields.DATETIME(stored=True),
+                modified_time = ID(stored=True),
-                indexed_time = fields.DATETIME(stored=True),
+                indexed_time = ID(stored=True),
                 
-                title = fields.TEXT(stored=True, field_boost=100.0),
+                title = TEXT(stored=True, field_boost=100.0),
+                url = ID(stored=True, unique=True),
                 
-                url = fields.ID(stored=True),
+                mimetype=ID(stored=True),
+                owner_email=ID(stored=True),
+                owner_name=TEXT(stored=True),
                 
-                mimetype = fields.TEXT(stored=True),
+                repo_name=TEXT(stored=True),
+                repo_url=ID(stored=True),
 
-                owner_email = fields.ID(stored=True),
+                github_user=TEXT(stored=True),
-                owner_name = fields.TEXT(stored=True),
-
-                # mainly for email threads, groups.io, hypothesis
-                group = fields.ID(stored=True),
-
-                repo_name = fields.TEXT(stored=True),
-                repo_url = fields.ID(stored=True),
-                github_user = fields.TEXT(stored=True),
-
-                tags = fields.KEYWORD(commas=True,
-                                      stored=True,
-                                      lowercase=True),
 
                 # comments only
-                issue_title = fields.TEXT(stored=True, field_boost=100.0),
+                issue_title=TEXT(stored=True, field_boost=100.0),
-                issue_url = fields.ID(stored=True),
+                issue_url=ID(stored=True),
                 
-                content = fields.TEXT(stored=True, analyzer=stemming_analyzer)
+                content=TEXT(stored=True, analyzer=stemming_analyzer)
         )
 
 
@@ -253,32 +243,24 @@ class Search:
             writer.delete_by_term('id',item['id'])
 
             # Index a plain google drive file
-            created_time = dateutil.parser.parse(item['createdTime'])
+            writer.add_document(
-            modified_time = dateutil.parser.parse(item['modifiedTime'])
+                    id = item['id'],
-            indexed_time = datetime.now().replace(microsecond=0)
+                    kind = 'gdoc',
-            try:
+                    created_time = item['createdTime'],
-                writer.add_document(
+                    modified_time = item['modifiedTime'],
-                        id = item['id'],
+                    indexed_time = datetime.now().replace(microsecond=0).isoformat(),
-                        kind = 'gdoc',
+                    title = item['name'],
-                        created_time = created_time,
+                    url = item['webViewLink'],
-                        modified_time = modified_time,
+                    mimetype = mimetype,
-                        indexed_time = indexed_time,
+                    owner_email = item['owners'][0]['emailAddress'],
-                        title = item['name'],
+                    owner_name = item['owners'][0]['displayName'],
-                        url = item['webViewLink'],
+                    repo_name='',
-                        mimetype = mimetype,
+                    repo_url='',
-                        owner_email = item['owners'][0]['emailAddress'],
+                    github_user='',
-                        owner_name = item['owners'][0]['displayName'],
+                    issue_title='',
-                        group='',
+                    issue_url='',
-                        repo_name='',
+                    content = content
-                        repo_url='',
+            )
-                        github_user='',
-                        issue_title='',
-                        issue_url='',
-                        content = content
-                )
-            except ValueError as e:
-                print(repr(e))
-                print(" > XXXXXX Failed to index Google Drive file \"%s\""%(item['name']))
 
 
         else:
@@ -332,7 +314,7 @@ class Search:
                 )
                 assert output == ""
             except RuntimeError:
-                print(" > XXXXXX Failed to index Google Drive document \"%s\""%(item['name']))
+                print(" > XXXXXX Failed to index document \"%s\""%(item['name']))
 
 
             # If export was successful, read contents of markdown
@@ -360,33 +342,24 @@ class Search:
             else:
                 print(" > Creating a new record")
 
-            try:
+            writer.add_document(
-                created_time = dateutil.parser.parse(item['createdTime'])
+                    id = item['id'],
-                modified_time = dateutil.parser.parse(item['modifiedTime'])
+                    kind = 'gdoc',
-                indexed_time = datetime.now()
+                    created_time = item['createdTime'],
-                writer.add_document(
+                    modified_time = item['modifiedTime'],
-                        id = item['id'],
+                    indexed_time = datetime.now().replace(microsecond=0).isoformat(),
-                        kind = 'gdoc',
+                    title = item['name'],
-                        created_time = created_time,
+                    url = item['webViewLink'],
-                        modified_time = modified_time,
+                    mimetype = mimetype,
-                        indexed_time = indexed_time,
+                    owner_email = item['owners'][0]['emailAddress'],
-                        title = item['name'],
+                    owner_name = item['owners'][0]['displayName'],
-                        url = item['webViewLink'],
+                    repo_name='',
-                        mimetype = mimetype,
+                    repo_url='',
-                        owner_email = item['owners'][0]['emailAddress'],
+                    github_user='',
-                        owner_name = item['owners'][0]['displayName'],
+                    issue_title='',
-                        group='',
+                    issue_url='',
-                        repo_name='',
+                    content = content
-                        repo_url='',
+            )
-                        github_user='',
-                        issue_title='',
-                        issue_url='',
-                        content = content
-                )
-            except ValueError as e:
-                print(repr(e))
-                print(" > XXXXXX Failed to index Google Drive file \"%s\""%(item['name']))
-
 
 
 
@@ -420,36 +393,31 @@ class Search:
                 issue_comment_content += comment.body.rstrip()
                 issue_comment_content += "\n"
 
-        # Now create the actual search index record.
+        # Now create the actual search index record
+        created_time = clean_timestamp(issue.created_at)
+        modified_time = clean_timestamp(issue.updated_at)
+        indexed_time = clean_timestamp(datetime.now())
+
         # Add one document per issue thread,
         # containing entire text of thread.
-
+        writer.add_document(
-        created_time = issue.created_at
+                id = issue.html_url,
-        modified_time = issue.updated_at
+                kind = 'issue',
-        indexed_time = datetime.now()
+                created_time = created_time,
-        try:
+                modified_time = modified_time,
-            writer.add_document(
+                indexed_time = indexed_time,
-                    id = issue.html_url,
+                title = issue.title,
-                    kind = 'issue',
+                url = issue.html_url,
-                    created_time = created_time,
+                mimetype='',
-                    modified_time = modified_time,
+                owner_email='',
-                    indexed_time = indexed_time,
+                owner_name='',
-                    title = issue.title,
+                repo_name = repo_name,
-                    url = issue.html_url,
+                repo_url = repo_url,
-                    mimetype='',
+                github_user = issue.user.login,
-                    owner_email='',
+                issue_title = issue.title,
-                    owner_name='',
+                issue_url = issue.html_url,
-                    group='',
+                content = issue_comment_content
-                    repo_name = repo_name,
+        )
-                    repo_url = repo_url,
-                    github_user = issue.user.login,
-                    issue_title = issue.title,
-                    issue_url = issue.html_url,
-                    content = issue_comment_content
-            )
-        except ValueError as e:
-            print(repr(e))
-            print(" > XXXXXX Failed to index Github issue \"%s\""%(issue.title))
 
 
 
@@ -479,8 +447,7 @@ class Search:
             print(" > XXXXXXXX Failed to find file info.")
             return
 
-
+        indexed_time = clean_timestamp(datetime.now())
-        indexed_time = datetime.now()
 
         if fext in MARKDOWN_EXTS:
             print("Indexing markdown doc %s from repo %s"%(fname,repo_name))
@@ -509,31 +476,24 @@ class Search:
             usable_url = "https://github.com/%s/blob/master/%s"%(repo_name, fpath)
 
             # Now create the actual search index record
-            try:
+            writer.add_document(
-                writer.add_document(
+                    id = fsha,
-                        id = fsha,
+                    kind = 'markdown',
-                        kind = 'markdown',
+                    created_time = '',
-                        created_time = None,
+                    modified_time = '',
-                        modified_time = None,
+                    indexed_time = indexed_time,
-                        indexed_time = indexed_time,
+                    title = fname,
-                        title = fname,
+                    url = usable_url,
-                        url = usable_url,
+                    mimetype='',
-                        mimetype='',
+                    owner_email='',
-                        owner_email='',
+                    owner_name='',
-                        owner_name='',
+                    repo_name = repo_name,
-                        group='',
+                    repo_url = repo_url,
-                        repo_name = repo_name,
+                    github_user = '',
-                        repo_url = repo_url,
+                    issue_title = '',
-                        github_user = '',
+                    issue_url = '',
-                        issue_title = '',
+                    content = content
-                        issue_url = '',
+            )
-                        content = content
-                )
-            except ValueError as e:
-                print(repr(e))
-                print(" > XXXXXX Failed to index Github markdown file \"%s\""%(fname))
-
-
 
         else:
             print("Indexing github file %s from repo %s"%(fname,repo_name))
@@ -541,29 +501,24 @@ class Search:
             key = fname+"_"+fsha
 
             # Now create the actual search index record
-            try:
+            writer.add_document(
-                writer.add_document(
+                    id = key,
-                        id = key,
+                    kind = 'ghfile',
-                        kind = 'ghfile',
+                    created_time = '',
-                        created_time = None,
+                    modified_time = '',
-                        modified_time = None,
+                    indexed_time = indexed_time,
-                        indexed_time = indexed_time,
+                    title = fname,
-                        title = fname,
+                    url = repo_url,
-                        url = repo_url,
+                    mimetype='',
-                        mimetype='',
+                    owner_email='',
-                        owner_email='',
+                    owner_name='',
-                        owner_name='',
+                    repo_name = repo_name,
-                        group='',
+                    repo_url = repo_url,
-                        repo_name = repo_name,
+                    github_user = '',
-                        repo_url = repo_url,
+                    issue_title = '',
-                        github_user = '',
+                    issue_url = '',
-                        issue_title = '',
+                    content = ''
-                        issue_url = '',
+            )
-                        content = ''
-                )
-            except ValueError as e:
-                print(repr(e))
-                print(" > XXXXXX Failed to index Github file \"%s\""%(fname))
 
 
 
@@ -577,42 +532,28 @@ class Search:
         Use a Github file API record to add a filename
         to the search index.
         """
-        if 'created_time' in d.keys() and d['created_time'] is not None:
+        indexed_time = clean_timestamp(datetime.now())
-            created_time = d['created_time']
-        else:
-            created_time = None
-
-        if 'modified_time' in d.keys() and d['modified_time'] is not None:
-            modified_time = d['modified_time']
-        else:
-            modified_time = None
-
-        indexed_time = datetime.now()
 
         # Now create the actual search index record
-        try:
+        writer.add_document(
-            writer.add_document(
+                id = d['permalink'],
-                    id = d['permalink'],
+                kind = 'emailthread',
-                    kind = 'emailthread',
+                created_time = '',
-                    created_time = created_time,
+                modified_time = '',
-                    modified_time = modified_time,
+                indexed_time = indexed_time,
-                    indexed_time = indexed_time,
+                title = d['subject'],
-                    title = d['subject'],
+                url = d['permalink'],
-                    url = d['permalink'],
+                mimetype='',
-                    mimetype='',
+                owner_email='',
-                    owner_email='',
+                owner_name=d['original_sender'],
-                    owner_name=d['original_sender'],
+                repo_name = '',
-                    group=d['subgroup'],
+                repo_url = '',
-                    repo_name = '',
+                github_user = '',
-                    repo_url = '',
+                issue_title = '',
-                    github_user = '',
+                issue_url = '',
-                    issue_title = '',
+                content = d['content']
-                    issue_url = '',
+        )
-                    content = d['content']
+
-            )
-        except ValueError as e:
-            print(repr(e))
-            print(" > XXXXXX Failed to index Groups.io thread \"%s\""%(d['subject']))
 
 
 
@@ -690,10 +631,10 @@ class Search:
                 full_items[f['id']] = f
             
             ## Shorter:
-            break
+            #break
-            ## Longer:
+            # Longer:
-            #if nextPageToken is None:
+            if nextPageToken is None:
-            #    break
+                break
 
 
         writer = self.ix.writer()
@@ -701,41 +642,34 @@ class Search:
         temp_dir = tempfile.mkdtemp(dir=os.getcwd())
         print("Temporary directory: %s"%(temp_dir))
 
-        try:
-
-            # Drop any id in indexed_ids
-            # not in remote_ids
-            drop_ids = indexed_ids - remote_ids
-            for drop_id in drop_ids:
-                writer.delete_by_term('id',drop_id)
 
 
-            # Update any id in indexed_ids
+        # Drop any id in indexed_ids
-            # and in remote_ids
+        # not in remote_ids
-            update_ids = indexed_ids & remote_ids
+        drop_ids = indexed_ids - remote_ids
-            for update_id in update_ids:
+        for drop_id in drop_ids:
-                # cop out
+            writer.delete_by_term('id',drop_id)
-                writer.delete_by_term('id',update_id)
-                item = full_items[update_id]
-                self.add_drive_file(writer, item, temp_dir, config, update=True)
-                count += 1
 
 
-            # Add any id not in indexed_ids
+        # Update any id in indexed_ids
-            # and in remote_ids
+        # and in remote_ids
-            add_ids = remote_ids - indexed_ids
+        update_ids = indexed_ids & remote_ids
-            for add_id in add_ids:
+        for update_id in update_ids:
-                item = full_items[add_id]
+            # cop out
-                self.add_drive_file(writer, item, temp_dir, config, update=False)
+            writer.delete_by_term('id',update_id)
-                count += 1
+            item = full_items[update_id]
+            self.add_drive_file(writer, item, temp_dir, config, update=True)
+            count += 1
+
+
+        # Add any id not in indexed_ids
+        # and in remote_ids
+        add_ids = remote_ids - indexed_ids
+        for add_id in add_ids:
+            item = full_items[add_id]
+            self.add_drive_file(writer, item, temp_dir, config, update=False)
+            count += 1
 
-        except Exception as e:
-            print("ERROR: While adding Google Drive files to search index")
-            print("-"*40)
-            print(repr(e))
-            print("-"*40)
-            print("Continuing...")
-            pass
 
         print("Cleaning temporary directory: %s"%(temp_dir))
         subprocess.call(['rm','-fr',temp_dir])
@@ -1140,7 +1074,7 @@ class Search:
         elif doctype=='issue':
             item_keys = ['title','repo_name','repo_url','url','created_time','modified_time']
         elif doctype=='emailthread':
-            item_keys = ['title','owner_name','url','created_time','modified_time']
+            item_keys = ['title','owner_name','url']
         elif doctype=='ghfile':
             item_keys = ['title','repo_name','repo_url','url']
         elif doctype=='markdown':
@@ -1157,7 +1091,11 @@ class Search:
             for r in results:
                 d = {}
                 for k in item_keys:
-                    d[k] = r[k]
+                    if k=='created_time' or k=='modified_time':
+                        #d[k] = r[k]
+                        d[k] = dateutil.parser.parse(r[k]).strftime("%Y-%m-%d")
+                    else:
+                        d[k] = r[k]
                 json_results.append(d)
 
         return json_results
@@ -1170,9 +1108,7 @@ class Search:
             query_string = " ".join(query_list)
             query = None
             if ":" in query_string:
-                query = QueryParser("content", self.schema)
+                query = QueryParser("content", self.schema).parse(query_string)
-                query.add_plugin(DateParserPlugin(free=True))
-                query = query.parse(query_string)
             elif len(fields) == 1 and fields[0] == "filename":
                 pass
             elif len(fields) == 2:
@@ -1180,12 +1116,9 @@ class Search:
             else:
                 # If the user does not specify a field,
                 # these are the fields that are actually searched
-                fields = ['title', 'content','owner_name','owner_email','url','created_date','modified_date']
+                fields = ['title', 'content','owner_name','owner_email','url']
             if not query:
-                query = MultifieldParser(fields, schema=self.ix.schema)
+                query = MultifieldParser(fields, schema=self.ix.schema).parse(query_string)
-                query.add_plugin(DateParserPlugin(free=True))
-                query = query.parse(query_string)
-                #query = MultifieldParser(fields, schema=self.ix.schema).parse(query_string) 
             parsed_query = "%s" % query
             print("query: %s" % parsed_query)
             results = searcher.search(query, terms=False, scored=True, groupedby="kind")

config_centillion.py

@@ -1,28 +0,0 @@
-config = {
-    "repositories" : [
-        "dcppc/project-management",
-        "dcppc/nih-demo-meetings",
-        "dcppc/internal",
-        "dcppc/organize",
-        "dcppc/dcppc-bot",
-        "dcppc/full-stacks",
-        "dcppc/design-guidelines-discuss",
-        "dcppc/dcppc-deliverables",
-        "dcppc/dcppc-milestones",
-        "dcppc/crosscut-metadata",
-        "dcppc/lucky-penny",
-        "dcppc/dcppc-workshops",
-        "dcppc/metadata-matrix",
-        "dcppc/data-stewards",
-        "dcppc/dcppc-phase1-demos",
-        "dcppc/apis",
-        "dcppc/2018-june-workshop",
-        "dcppc/2018-july-workshop",
-        "dcppc/2018-august-workshop",
-        "dcppc/2018-september-workshop",
-        "dcppc/design-guidelines",
-        "dcppc/2018-may-workshop",
-        "dcppc/centillion"
-    ]
-}
-

groupsio_util.py

@@ -1,7 +1,5 @@
 import requests, os, re
 from bs4 import BeautifulSoup
-import dateutil.parser
-import datetime
 
 class GroupsIOException(Exception):
     pass
@@ -66,7 +64,7 @@ class GroupsIOArchivesCrawler(object):
 
             ## Short circuit
             ## for debugging purposes
-            break
+            #break
 
         return subgroups
 
@@ -253,7 +251,7 @@ class GroupsIOArchivesCrawler(object):
             subject = soup.find('title').text
 
             # Extract information for the schema:
-            # - permalink for thread (done above)
+            # - permalink for thread (done)
             # - subject/title (done)
             # - original sender email/name (done)
             # - content (done)
@@ -268,35 +266,11 @@ class GroupsIOArchivesCrawler(object):
                     pass
                 else:
                     # found an email!
-                    # this is a maze, not amazing.
+                    # this is a maze, thanks groups.io
-                    # thanks groups.io!
                     td = tr.find('td')
-
+                    divrow = td.find('div',{'class':'row'}).find('div',{'class':'pull-left'})
-                    sender_divrow = td.find('div',{'class':'row'})
-                    sender_divrow = sender_divrow.find('div',{'class':'pull-left'})
                     if (i+1)==1:
-                        original_sender = sender_divrow.text.strip()
+                        original_sender = divrow.text.strip()
-
-                    date_divrow = td.find('div',{'class':'row'})
-                    date_divrow = date_divrow.find('div',{'class':'pull-right'})
-                    date_divrow = date_divrow.find('font',{'class':'text-muted'})
-                    date_divrow = date_divrow.find('script').text
-                    try:
-                        time_seconds = re.search(' [0-9]{1,} ',date_divrow).group(0)
-                        time_seconds = time_seconds.strip()
-                        # Thanks groups.io for the weird date formatting
-                        time_seconds = time_seconds[:10]
-                        mmicro_seconds = time_seconds[10:]
-                        if (i+1)==1:
-                            created_time  = datetime.datetime.utcfromtimestamp(int(time_seconds))
-                            modified_time = datetime.datetime.utcfromtimestamp(int(time_seconds))
-                        else:
-                            modified_time = datetime.datetime.utcfromtimestamp(int(time_seconds))
-
-                    except AttributeError:
-                        created_time = None
-                        modified_time = None
-
                     for div in td.find_all('div'):
                         if div.has_attr('id'):
 
@@ -325,10 +299,7 @@ class GroupsIOArchivesCrawler(object):
 
             thread = {
                     'permalink' : permalink,
-                    'created_time' : created_time,
-                    'modified_time' : modified_time,
                     'subject' : subject,
-                    'subgroup' : subgroup_name,
                     'original_sender' : original_sender,
                     'content' : full_content
             }
@@ -353,13 +324,11 @@ class GroupsIOArchivesCrawler(object):
 
         results = []
         for row in rows:
-            # This is where we extract
+            # We don't care about anything except title and ugly link
-            # a list of thread titles 
-            # and corresponding links.
             subject = row.find('span',{'class':'subject'})
             title = subject.get_text()
             link = row.find('a')['href']
-
+            #print(title)
             results.append((title,link))
 
         return results

static/feedback.js

@@ -1,133 +0,0 @@
-// submitting form with modal:
-// https://stackoverflow.com/a/29068742
-//
-// closing a bootstrap modal with submit button:
-// https://stackoverflow.com/a/33478107
-//
-// flask post data as json:
-// https://stackoverflow.com/a/16664376
-
-/* this function is called when the user submits
- * the feedback form. it submits a post request
- * to the flask server, which squirrels away the
- * feedback in a file.
- */
-function submit_feedback() {
-    // this function is called when submit button clicked
-    // algorithm:
-    // - check if text box has content
-    // - check if happy/sad filled out
-
-    var smile_active = $('#modal-feedback-smile-div').hasClass('smile-active');
-    var frown_active = $('#modal-feedback-frown-div').hasClass('frown-active');
-    if( !( smile_active || frown_active ) ) {
-        alert('Please pick the smile or the frown.')
-    } else if( $('#modal-feedback-textarea').val()=='' ) {
-        alert('Please provide us with some feedback.')
-    } else {
-        var user_sentiment = '';
-        if(smile_active) {
-            user_sentiment = 'smile';
-        } else {
-            user_sentiment = 'frown';
-        }
-        var escaped_text = $('#modal-feedback-textarea').val();
-
-        // prepare form data 
-        var data = {
-            sentiment : user_sentiment,
-            content : escaped_text
-        };
-        // post the form. the callback function resets the form
-        $.post("/feedback", 
-            data, 
-            function(response) {
-                $('#myModal').modal('hide');
-                $('#myModalForm')[0].reset();
-                add_alert(response);
-                frown_unclick();
-                smile_unclick();
-        });
-    }
-}
-
-
-function add_alert(response) {
-    str = ""
-    str += '<div id="feedback-messages-container" class="container">';
-
-    if (response['status']=='ok') {
-        // if status is ok, use alert-success
-        str += '    <div id="feedback-messages-alert" class="alert alert-success alert-dismissible fade in">';
-    } else {
-        // otherwise use alert-danger
-        str += '    <div id="feedback-messages-alert" class="alert alert-danger alert-dismissible fade in">';
-    }
-
-    str += '        <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>';
-    str += '        <div id="feedback-messages-contianer" class="container-fluid">';
-    str += '            <div id="feedback-messages-div" class="co-xs-12">';
-    str += '                <p>'
-    str += response['message'];
-    str += '                </p>';
-    str += '            </div>';
-    str += '    </div>';
-    str += '</div>';
-    $('div#messages').append(str);
-}
-
-
-/* for those particularly wordy users... limit feedback to 1000 chars */
-function cool_it() { 
-    if($('#modal-feedback-textarea').val().length > 1100 ){
-        $('#modal-too-long').show();
-    } else {
-        $('#modal-too-long').hide();
-    }
-}
-
-/* smiley functions */
-function smile_click() {
-    $('#modal-feedback-smile-div').addClass('smile-active');
-    $('#modal-feedback-smile-icon').addClass('smile-active');
-}
-function frown_click() {
-    $('#modal-feedback-frown-div').addClass('frown-active');
-    $('#modal-feedback-frown-icon').addClass('frown-active');
-}
-function smile_unclick() {
-    $('#modal-feedback-smile-div').removeClass('smile-active');
-    $('#modal-feedback-smile-icon').removeClass('smile-active');
-}
-function frown_unclick() {
-    $('#modal-feedback-frown-div').removeClass('frown-active');
-    $('#modal-feedback-frown-icon').removeClass('frown-active');
-}
-
-function smile() {
-    frown_unclick();
-    smile_click();
-}
-function frown() { 
-    smile_unclick();
-    frown_click();
-}
-
-
-/* for those particularly wordy users... limit feedback to 1100 chars */
-// how to check n characters in a textarea
-// https://stackoverflow.com/a/19934613
-/*
-$(document).ready(function() {
-
-    $('#modal-feedback-textarea').on('change',function(event) {
-        if($('#modal-feedback-textarea').val().length > 1100 ){
-            $('#modal-too-long').show();
-        } else {
-            $('#modal-too-long').hide();
-        }
-    });
-
-}
-*/
-

static/style.css

@@ -1,61 +1,3 @@
-#modal-too-long {
-    visibility: hidden;
-}
-
-/* feedback smileys */
-#modal-feedback-smile-icon,
-#modal-feedback-frown-icon {
-    padding-left: 100px;
-    padding-right: 100px;
-    padding-top: 20px;
-    padding-bottom: 20px;
-}
-
-div.smile-active {
-    background-color: #2b2;
-}
-i.smile-active {
-    color: #fff;
-}
-div.frown-active {
-    background-color: #b22;
-}
-i.frown-active {
-    color: #fff;
-}
-
-/* feedback text area */
-#modal-feedback-textarea {
-    width: 100%;
-}
-
-/* feedback buttons */
-button.close {
-    font-size: 35px;
-}
-button#submit-feedback-btn {
-    width: 250px;
-}
-
-button#feedback:hover {
-    opacity: 1.0;
-    filter: alpha(opacity=100); /* For IE8 and earlier */
-}
-
-button#feedback {
-    opacity: 0.5;
-    filter: alpha(opacity=50); /* For IE8 and earlier */
-
-    width: 180px;
-    height: 50px;
-    position: fixed; 
-    z-index: 999;
-    right: 120px;
-    bottom: 10px;
-}
-
-
-/* search results table */
 td#search-results-score-col,
 td#search-results-type-col {
     width: 100px;

templates/403.html

@@ -1,5 +1,4 @@
 {% extends "layout.html" %}
-{% set active_page = "403" %}
 {% block body %}
 
 <div class="container">

templates/404.html

@@ -1,5 +1,4 @@
 {% extends "layout.html" %}
-{% set active_page = "404" %}
 {% block body %}
 
 <div class="container">

templates/banner.html

@@ -1,25 +0,0 @@
-<div class="container" id="banner-container">
-
-    {#
-    banner image
-    #}
-    <div class="row" id="banner-row">
-        <div class="col12sm" id="banner-col">
-            <center>
-                <a id="banner-a" href="{{ url_for('search')}}?query=&fields=">
-                    <img id="banner-img" src="{{ url_for('static', filename='centillion_white.png') }}">
-                </a>
-            </center>
-        </div>
-    </div>
-    {% if config['TAGLINE'] %}
-    <div class="row" id="tagline-row">
-        <div class="col12sm" id="tagline-col">
-            <center>
-                    <h2 id="tagline-tagline"> {{config['TAGLINE']}} </h2>
-            </center>
-        </div>
-    </div>
-    {% endif %}
-
-</div>

templates/controlpanel.html

@@ -1,5 +1,4 @@
 {% extends "layout.html" %}
-{% set active_page = "control_panel" %}
 {% block body %}
 
 <hr />

templates/flashed_messages.html

@@ -1,14 +0,0 @@
-<div id="messages">
-{% with messages = get_flashed_messages() %}
-    {% if messages %}
-        <div class="container" id="flashed-messages-container">
-            <div class="alert alert-success alert-dismissible fade in">
-                <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
-                    {% for message in messages %}
-                        <p class="lead">{{ message }}</p>
-                    {% endfor %}
-            </div>
-        </div>
-    {% endif %}
-{% endwith %}
-</div>

templates/landing.html

@@ -1,5 +1,4 @@
 {% extends "layout.html" %}
-{% set active_page = "landing" %}
 {% block body %}
 <div class="container">
     <div class="row">

templates/layout.html

@@ -10,7 +10,6 @@
 <script src="{{ url_for('static', filename='bootstrap.min.js') }}"></script>
 <script src="{{ url_for('static', filename='master_list.js') }}"></script>
 <script src="{{ url_for('static', filename='search_list.js') }}"></script>
-<script src="{{ url_for('static', filename='feedback.js') }}"></script>
 
 
 {# ########## dataTables plugin ############ #}
@@ -24,43 +23,52 @@
 
 
 
-<div id="master-div">
+{# ########## github fork corner ############ #}
 
-    {#
+<div>
-    flashed messages
-    #}
-    {% include "flashed_messages.html" %}
 
-    {#
+    {% with messages = get_flashed_messages() %}
-    banner image
+    {% if messages %}
-    #}
+    <div class="container">
-    {% include "banner.html" %}
+        <div class="alert alert-success alert-dismissible">
+            <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+            <ul class=flashes>
+                {% for message in messages %}
+                <li>{{ message }}</li>
+                {% endfor %}
+            </ul>
+        </div>
+    </div>
+    {% endif %}
+    {% endwith %}
 
-    {#
+    <div class="container">
-    feedback modal
-    #}
-    {% include "modal.html" %}
     
-    {% block body %}{% endblock %}
+        {#
+        banner image
+        #}
+        <div class="row">
+            <div class="col12sm">
+                <center>
+                    <a href="{{ url_for('search')}}?query=&fields=">
+                    <img src="{{ url_for('static', filename='centillion_white.png') }}">
+                    </a>
+                    {#
+                    need a tag line
+                    #}
+                    {% if config['TAGLINE'] %}
+                        <h2><a href="{{ url_for('search')}}?query=&fields=">
+                            {{config['TAGLINE']}}
+                        </a></h2>
+                    {% endif %}
+                </center>
+            </div>
+        </div>
     
-</div>
-
-{% if active_page=="search" or active_page=="master_list" %}
-
-    {# feedback button #}
-    <button id="feedback" type="button" 
-                          data-toggle="modal"
-                          data-target="#myModal"
-        class="btn btn-lg">Send Feedback</button>
-    
-    {# vertical spacing before the bottom, b/c of button #}
-    <div id="footer-whitespace" class="container">
-        <p>&nbsp;</p>
-        <p>&nbsp;</p>
-        <p>&nbsp;</p>
     </div>
 
-{% endif %}
+    {% block body %}{% endblock %}
+</div>
 
-<a id="github-corner" href="https://github.com/dcppc/centillion" class="github-corner" aria-label="View source on Github"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
+<a href="https://github.com/dcppc/centillion" class="github-corner" aria-label="View source on Github"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>
 

templates/masterlist.html

@@ -1,5 +1,4 @@
 {% extends "layout.html" %}
-{% set active_page = "master_list" %}
 {% block body %}
 
 <hr />

templates/modal.html

@@ -1,51 +0,0 @@
-<div class="modal fade" id="myModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel">
-
-  <form id="myModalForm" method="post">
-
-    <div class="modal-dialog" role="document">
-      <div id="myModal-content" class="modal-content">
-        <div id="myModal-header" class="modal-header">
-          <button type="button" class="close" data-dismiss="modal" aria-label="Close">
-              <span aria-hidden="true">&times;</span>
-          </button>
-          <h4 class="modal-title" id="myModalLabel">
-          Send us feedback!
-          </h4>
-        </div>
-        <div id="myModal-body" class="modal-body">
-          <div id="modal-feedback-smile-frown-container" class="container-fluid">
-              <div id="modal-feedback-smile-div" class="col-xs-6 text-center" 
-                  onClick="smile()">
-                  <i id="modal-feedback-smile-icon" class="fa fa-smile-o fa-4x" aria-hidden="true"></i>
-              </div>
-              <div id="modal-feedback-frown-div" class="col-xs-6 text-center" 
-                  onClick="frown()">
-                  <i id="modal-feedback-frown-icon" class="fa fa-frown-o fa-4x" aria-hidden="true"></i>
-              </div>
-          </div>
-          <div class="container-fluid">
-              <p>&nbsp;</p>
-          </div>
-          <div id="modal-feedback-textarea-container" class="container-fluid">
-              <textarea id="modal-feedback-textarea" rows="6"></textarea>
-          </div>
-          <div id="modal-too-long" class="container-fluid" >
-              <p id="modal-too-long-text" class="lead">Please limit the length of your feedback. Thank you in advance!</p>
-          </div>
-        </div>
-        <div id="myModal-footer" class="modal-footer">
-            <div class="text-center">
-              <button id="submit-feedback-btn" type="button" 
-                  onClick="submit_feedback()"
-                  class="btn btn-lg btn-primary">
-                  Send
-              </button>
-            </div>
-        </div>
-      </div>
-    </div>
-
-  </form>
-
-</div>        
-

templates/search.html

@@ -1,8 +1,7 @@
 {% extends "layout.html" %}
-{% set active_page = "search" %}
 {% block body %}
 
-<div id="search-bar-container" class="container">
+<div class="container">
     <div class="row">
 
         <div class="col-xs-12">
@@ -13,11 +12,7 @@
                     <p><button id="the-big-one" type="submit" style="font-size: 20px; padding: 10px; padding-left: 50px; padding-right: 50px;" 
                         value="search" class="btn btn-primary">Search</button>
                     </p>
-
+                    <p><a href="{{ url_for('search')}}?query=&fields=">[clear all results]</a>
-                    {% if parsed_query %}
-                        <p><a href="{{ url_for('search')}}?query=&fields=">[clear all results]</a>
-                    {% endif %}
-
                     </p>
                 </form>
             </center>
@@ -25,11 +20,18 @@
     </div>
 </div>
 
-<div style="height: 20px;"><p>&nbsp;</p></div>
+<div class="container">
-
-<div id="info-bars-container" class="container">
     <div class="row">
 
+        {% if directories %}
+        <div class="col-xs-12 info directories-cloud">
+            <b>File directories:</b> 
+            {% for d in directories %}
+                <a href="{{url_for('search')}}?query={{d|trim}}&fields=filename">{{d|trim}}</a>
+            {% endfor %}
+        </div>
+        {% endif %}
+
         <ul class="list-group">
 
             {% if config['SHOW_PARSED_QUERY'] and parsed_query %}
@@ -44,9 +46,6 @@
                 </li>
             {% endif %}
 
-
-            {# use "if parsed_query" to check if this is 
-               a new search or search results #}
             {% if parsed_query %}
                 <li  class="list-group-item">
                     <div class="container-fluid">
@@ -60,7 +59,6 @@
                 </li>
             {% endif %}
 
-
             <li  class="list-group-item">
                     <div class="container-fluid">
                         <div class="row">
@@ -101,7 +99,7 @@
 </div>
 
 {% if parsed_query %}
-<div id="search-results-container" class="container">
+<div class="container">
     <div class="row">
         <table id="search-results" class="table">
             <thead id="search-results-header">
@@ -128,21 +126,44 @@
                             {% if e.kind=="gdoc" %}
                                 {% if e.mimetype=="document" %}
                                     <p><small>Drive Document</small</p>
+                                    <!--
+                                    <i class="fa fa-google fa-2x"></i>
+                                    <i class="fa fa-file-text fa-2x"></i>
+                                    -->
                                 {% else %}
                                     <p><small>Drive File</small</p>
+                                    <!--
+                                    <i class="fa fa-google fa-2x"></i>
+                                    <i class="fa fa-file-o fa-2x"></i>
+                                    -->
                                 {% endif %}
 
                             {% elif e.kind=="issue" %}
                                 <p><small>Issue</small</p>
+                                <!--
+                                <i class="fa fa-github fa-2x"></i>
+                                <i class="fa fa-question fa-2x"></i>
+                                -->
 
                             {% elif e.kind=="ghfile" %}
                                 <p><small>Github File</small</p>
+                                <!--
+                                <i class="fa fa-github fa-2x"></i>
+                                <i class="fa fa-file-o fa-2x"></i>
+                                -->
 
                             {% elif e.kind=="markdown" %}
                                 <p><small>Github Markdown</small</p>
+                                <!--
+                                <i class="fa fa-github fa-2x"></i>
+                                <i class="fa fa-file-text-o fa-2x"></i>
+                                -->
 
                             {% elif e.kind=="emailthread" %}
                                 <p><small>Email Thread</small</p>
+                                <!--
+                                <i class="fa fa-envelope-o fa-2x"></i>
+                                -->
 
                             {% else %}
                                 <p><small>Unknown</small</p>