#!/usr/bin/env python3

# SPDX-License-Identifier: LGPL-3.0-or-later
# Copyright (C) 2023 Chao Peng
'''
    A script to compare two detector parameter tables.
    Report inconsistent numbers, missing components/columns, or other errors.
'''

import os
import json
import argparse
import numpy as np
import pandas as pd

pd.set_option('display.max_rows', 500)

# key_cols + value_cols are required columns
key_cols = [
        'Region',
        'Component',
        'Sub-Component',
        ]
value_cols = [
        'Length (cm)',
        'Inner Radius (cm)',
        'Outer Radius (cm)',
        'Offset from Center (cm)',
        'Physical Start (cm)',
        'Physical End (cm)',
        ]

# read parameter table and do a little bit formatting
def read_par_table(path, **kwargs):
    # read and format
    if path.endswith('html') or path.endswith('htm'):
        dft = pd.read_html(path, **kwargs)[0]
    else:
        dft = pd.read_csv(path, **kwargs)
    dft.columns = [i.replace(' \n', ' ').replace('\n', ' ').strip() for i in dft.columns.astype(str)]
    unnamed_cols = [i for i in dft.columns if i.startswith('Unnamed')]
    dft = dft.drop(columns=unnamed_cols)
    for col in dft.columns:
        if pd.api.types.is_string_dtype(dft[col].dtype):
            dft.loc[:, col] = dft[col].str.replace('\n', ' ').str.replace('  ', ' ').str.strip()

    missing_required_cols = [c for c in key_cols + value_cols if c not in dft.columns]
    if len(missing_required_cols) > 0:
        print('Error: required columns do not exist in \"{}\": {}'.format(path, missing_required_cols))
        exit(-1)

    dft.loc[:, 'Region'] = dft['Region'].fillna(method='ffill')
    dft.loc[:, 'Component'] = dft['Component'].fillna(method='ffill')
    dft.loc[:, 'Sub-Component'] = dft['Sub-Component'].fillna('')
    # build a key (multi-indexing also works)
    comp = dft['Region'] + '_' + dft['Component']
    sub_comp = dft['Sub-Component'].copy()
    sub_comp.loc[sub_comp.str.len() > 0] = '_' + sub_comp.astype(str)
    dft.loc[:, 'key'] = (comp + sub_comp).str.replace(' ', '_').str.lower()
    return dft.set_index('key', drop=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            prog='compare_detector_parameter_table',
            description = 'A python script to compare two detector parameter tables.'
                        + '\n       ' # 7 spaces for 'usage: '
                        + r'It adapts the format as of 2023/10 (see https://eic.jlab.org/Menagerie).'
            )
    parser.add_argument(
            'det_table',
            help='Path or url to the DETECTOR parameter table (csv or html).'
            )
    parser.add_argument(
            'sim_table',
            help='Path or url to the SIMULATION parameter table (csv or html).'
            )
    parser.add_argument(
            '--template',
            default='https://raw.githubusercontent.com/eic/epic/main/templates/DetectorParameterTable.csv.jinja2',
            help='Path or url to the template file of the detector paramter table'
            )
    parser.add_argument(
            '--det-encoding', default='ISO-8859-1',
            help='encoding for the detector parameter table.'
            )
    parser.add_argument(
            '--sim-encoding', default='utf-8',
            help='encoding for the simulation parameter table.'
            )
    parser.add_argument(
            '-v', '--verbose', action='store_true',
            help='enable to print out the report.'
            )
    parser.add_argument(
            '-o', '--output', default='par_table_report.json',
            help='output path for the report.'
            )
    args = parser.parse_args()

    dfd = read_par_table(args.det_table, encoding=args.det_encoding)
    # ignore alternatives
    dfd = dfd[~dfd['Region'].str.contains('ALTERNATIVE')]
    dfs = read_par_table(args.sim_table, encoding=args.sim_encoding)

    try:
        temp = read_par_table(args.template)
    except:
        print('Failed to load parameter table template from \"{}\".'.format(args.template))
        print('Report will not include information about the simulation variable names.')
        temp = pd.DataFrame()

    # check components
    report = dict()
    matched_keys = np.intersect1d(dfd.index, dfs.index)
    missing_keys = [k for k in dfd.index if k not in matched_keys]
    extra_keys = [k for k in dfs.index if k not in matched_keys]
    df_mis = dfd.loc[missing_keys, ['Region', 'Component', 'Sub-Component']].reset_index(drop=True)
    df_ext = dfs.loc[extra_keys, ['Region', 'Component', 'Sub-Component']].reset_index(drop=True)
    dfdm = dfd.loc[matched_keys, key_cols + value_cols]
    dfsm = dfs.loc[:, key_cols + value_cols]
    try:
        dftm = temp.loc[:, key_cols + value_cols]
    except:
        dftm = pd.DataFrame()

    comps = []
    total_counts = np.zeros(3, dtype=int)
    # check component values
    for k, drow in dfdm.iterrows():
        srow = dfsm.loc[k].squeeze()
        try:
            tvars = dftm.loc[k].squeeze().to_dict()
        except:
            tvars = dict()
        # check value
        comp = drow[key_cols].to_dict()
        missing_vals, wrong_vals, correct_vals = {}, {}, {}
        # print(tvars)
        for vcol in value_cols:
            # print(k, vcol)
            tvar = str(tvars.get(vcol, 'Not Found')).replace('nan', 'Empty')
            try:
                dval = float(drow[vcol])
            except:
                dval = np.nan
            try:
                sval = float(srow[vcol])
            except:
                sval = np.nan

            if np.any(np.isnan([sval, dval])):
                missing_vals[vcol] = dict(det=dval, sim=sval, template_var=tvar)
            elif not np.isclose(sval, dval):
                wrong_vals[vcol] = dict(det=dval, sim=sval, template_var=tvar)
            else:
                correct_vals[vcol] = dict(det=dval, sim=sval, template_var=tvar)

        comp['Mismatched'] = wrong_vals
        total_counts[0] += len(wrong_vals)
        comp['Missing'] = missing_vals
        total_counts[1] += len(missing_vals)
        comp['Correct'] = correct_vals
        total_counts[2] += len(correct_vals)
        comps.append(comp)

    # printout reports
    if args.verbose:
        print('These det table components are missing in the sim table:')
        print(df_mis)

        print('These sim table components are not found in the det table:')
        print(df_ext)

        print('These components are checked:')

        for comp in comps:
            subdet = pd.DataFrame(columns=['det', 'sim', 'template_var', 'stat'], index=value_cols)
            print('{}, {}, {}: '.format(comp['Region'], comp['Component'], comp['Sub-Component']))
            for stat in ['Mismatched', 'Missing', 'Correct']:
                for key, val in comp[stat].items():
                    subdet.loc[key, 'stat'] = stat
                    for k, v in val.items():
                        subdet.loc[key, k] = v
            print(subdet)
            print(' ')

    # save reports to a json file
    report['stats'] = dict(
            checked_columns=value_cols,
            table_components=dict(
                detector=dict(
                    total=dfd.shape[0],
                    matched=dfdm.shape[0],
                    missing=df_mis.shape[0],),
                simulation=dict(
                    total=dfs.shape[0],
                    matched=dfdm.shape[0],
                    missing=df_ext.shape[0],)
                ),
            component_values=dict(
                checked_components=dfdm.shape[0],
                checked_values=int(np.sum(total_counts)),
                mismatched=int(total_counts[0]),
                missing=int(total_counts[1]),
                correct=int(total_counts[2]),
                ),
            )
    report['results'] = dict(
            checked_components=comps,
            missing_components=df_mis.to_dict('records'),
            extra_components=df_ext.to_dict('records')
            )
    with open(args.output, 'w') as f:
        json.dump(report, f, indent=4)
    print('Comparison reports saved to \"{}\".'.format(args.output))
