#!/usr/bin/env python

import os, re
from pathlib import Path
import pandas as pd
import tldextract
from bs4 import BeautifulSoup

# move to directory of the current file
os.chdir(os.path.dirname(os.path.abspath(__file__)))

# relative directory of the raw data as from the mirorring of the blog
dir = './path/to/blog'

# construct a data frame for each file in this directory
data = []
file_count = 0
file_fail_count = 0
classes = []
# loop through all blog posts using the glob pattern
for fn in Path(dir).glob('20[0-9][0-9]/[0-9][0-9]/[0-9][0-9]/*/index.html'):
    if not fn.is_file(): continue
    # print(fn)
    soup = BeautifulSoup(open(fn, encoding='utf8'), 'html.parser')
    # use html element with id=main as context
    if soup.find(id='main') is None:
      file_fail_count += 1
      print('xxx does not have #main xxx')
      continue
    file_count += 1
    # every non-relative link that we find is added to the dataset using
    # source: file name (without extension) where the link was found
    # url: the used URL
    # main: the domain and suffix of the url (e.g. wikipedia.org)
    # text: the clickable text of the link
    data_len = len(data)
    for link in soup.find(id='main').find_all('a', href = re.compile('^((https://)|(http://))')):
      # parse the domain
      ext = tldextract.extract(link.get('href'))
      # exclude same-domain links
      if ext.domain == 'klimatsans': continue
      # exclude social links
      if link.has_attr('class') and any(c.startswith('synved-social') for c in link['class']): continue
      # exclude: profile links in comments, pingbacks when other blogs link
      if link.has_attr('class') and 'url' in link['class'] and ( \
          link.find_parent('div', class_='comment-author') or \
          link.find_parent('li', class_='pingback')): continue
      print('--', link.get('href'))
      ## append to dataframe
      data.append({
        'source' : str(Path(fn)).replace('\\', '/').replace('../DATA_LINUX/blogs/', ''),
        'date'   : re.search( "20[12][0-9]-[01][0-9]-[0-3][0-9]", str(fn).replace('\\', '-')).group(),
        'url'    : link.get('href'),
        'domain' : link.get('href').split('/')[2].replace('www.', ''),
        'main'   : ext.registered_domain,
        'text'   : link.text
        })
      if link.has_attr('class'):
        print('--', link['class'])
        classes.extend(link['class'])

    if len(data)-data_len > 0: print('-- new links: ', len(data)-data_len)

print('Total files:', file_count, ' / ', file_fail_count, 'failed')
print('Total links:', len(data))
print(classes)
# save in a new TSV file
df = pd.DataFrame.from_records(data)
df.to_csv('./step3_all_hyperlinks.tsv', sep='\t')