import pandas as pd
import numpy as np

from os import listdir
from os.path import isfile, join

def process_col_names(table):
    
    '''
    Does the following on all columns:
    1) Converts names to lowercase
    2) Removes extra spaces 
    '''
    
    table.columns = table.columns.str.lower()
    
    table.columns = pd.Index([" ".join(col.split()) for col in table.columns])
    
    return table

def combine_html_files( html_path):
    
    '''
    Combines all html files in a directory in to single dataframe
    
    html_path : path for directory
    '''
    
    tables = []
    df_collection = []
    
    # Gather all html files in directory
    html_files = [f for f in listdir(html_path) if (f.endswith(".htm") or f.endswith(".html"))]
    print("file_count:",len(html_files))
    
    # Iterate over all files 
    for file in html_files:
        
        final_path = html_path + file
        
        try: 
            df = pd.read_html(final_path, header=0)
            
            for tabl in df:
                
                for col in tabl.columns:
                    if len(col) > 100:
                        print("Path where len(col) > 100:",final_path[final_path.find("BASE"):])
                        print("----------------------------------")
                        break
                
                # EXTRACT LOACTION
                tabl_loc = final_path[final_path.find("BASE"):]
                tabl['file location'] = tabl_loc
                
                tables.append( process_col_names( tabl ) )
                #table_index = int(x)-1
            
            result = pd.concat(tables, sort = True)
            df_collection.append(result)

        except ValueError:
            print("Value error for -->", file)
            print("----------------------------------")
    
    final_df = pd.concat(df_collection, sort = True, ignore_index= True) 
    
    print("Final shape", final_df.shape)
    print("Final file count", len(final_df['file location'].unique()) )
    
    return final_df