"""Transform raw New York City, NY data to prepared CSV files."""

import pandas as pd
import numpy as np
import geopandas as gpd
import logging
# from tqdm import tqdm
import click
import yaml
from functools import reduce
import datetime
import re
import json

pd.options.mode.chained_assignment = None

def get_month_as_str_col(df, date_col):
    """Generate a pandas series of the month as a string in YYYY-MM format from the date column.

    Parameters
    ----------
    df : pandas df
        Pandas dataframe
    date_col : str
        Name of the column in the dataframe containing the timestamp

    Returns
    -------
    pandas series
        Pandas series with the month as string in YYYY-MM format
    """
    return df[date_col].dt.to_period("M").astype(str)

def load_evictions_data(data_path, date_col, min_year, max_year, create_geoid = True):
    """Load the evictions data, filter for the correct years, and add a column that
    contains the month.

    Parameters
    ----------
    data_path : str
        Path to raw mortgage foreclosures data in CSV format
    date_col : str
        The eviction date column from which to extract the month and year
    min_year : int
        Earliest year to include in analysis
    max_year : int
        Latest year to include in analysis

    Returns
    -------
    pandas df
        Dataframe containing evictions data
    """
    ev_raw = (
        pd.read_csv(
            data_path,
            parse_dates=[date_col],
            infer_datetime_format=True,
        )
        .dropna(how="all")
        .drop_duplicates()
    )
    ev_raw["year"] = ev_raw[date_col].dt.year
    ev_df = ev_raw[(ev_raw.year >= min_year) & (ev_raw.year <= max_year)]
    ev_df["year"] = ev_df["year"].astype(int)
    ev_df["month"] = get_month_as_str_col(ev_df, date_col)

    if create_geoid == True:
        # Convert to strings
        ev_df["state_code"] = ev_df["state_code"].astype(str).replace(r'\.0', '', regex = True)
        ev_df["county_code"] = ev_df["county_code"].astype(str).replace(r'\.0', '', regex = True)
        ev_df["tract_code"] = ev_df["tract_code"].astype(str).replace(r'\.0', '', regex = True)
        # Add zeroes as necessary
        ev_df["state_code"] = ev_df["state_code"].apply(lambda x: (((2-len(x)) * "0") + x) if len(x) < 2 else x)
        ev_df["county_code"] = ev_df["county_code"].apply(lambda x: (((3-len(x)) * "0") + x) if len(x) < 3 else x)
        ev_df["tract_code"] = ev_df["tract_code"].apply(lambda x: (((6-len(x)) * "0") + x) if len(x) < 6 else x)
        # Finally, concat everything
        ev_df["GEOID"] = (
            ev_df["state_code"] + ev_df["county_code"] + ev_df["tract_code"]
        )

    return ev_df

def aggregate_evictions_using_rate_estimates(data_frame, eviction_rates_path, estimate_var = "evictions", time_group = "year"):

    if estimate_var == "evictions":
        # Aggregate all rows to count eviction filings
        filings_df = data_frame.groupby(by=["GEOID", time_group]).size().to_frame('eviction-filings').reset_index()

        # Loading eviction rates by tract, using Eviction Lab data for 2014-2016
        eviction_rates_by_tract = pd.read_csv(eviction_rates_path)
        overall_eviction_rate = sum(eviction_rates_by_tract["Total_Evictions"])/sum(eviction_rates_by_tract["Total_Filings"])

        eviction_rates_by_tract = eviction_rates_by_tract[["GEOID", "eviction-rate"]]
        eviction_rates_by_tract["GEOID"] = eviction_rates_by_tract["GEOID"].astype(str)

        # Returning final data frame
        evictions_final = filings_df.merge(eviction_rates_by_tract, how='left', on='GEOID')
        evictions_final["eviction-rate"].fillna(overall_eviction_rate, inplace=True)
        evictions_final["evictions"] = evictions_final["eviction-filings"] * evictions_final["eviction-rate"]
        evictions_final = evictions_final[["GEOID", time_group, "eviction-filings", "evictions", "eviction-rate"]]

        # Filtering to desired county's GEOIDs only
        # evictions_final = evictions_final[evictions_final.GEOID.str.contains('^12057')]

    if estimate_var == "filings":
        # Aggregate all rows to count eviction filings
        filings_df = data_frame.groupby(by=["GEOID", time_group]).size().to_frame('evictions').reset_index()

        # Loading eviction rates by tract, using Eviction Lab data for 2014-2016
        eviction_rates_by_tract = pd.read_csv(eviction_rates_path)
        overall_eviction_rate = sum(eviction_rates_by_tract["Total_Evictions"])/sum(eviction_rates_by_tract["Total_Filings"])

        eviction_rates_by_tract = eviction_rates_by_tract[["GEOID", "eviction-rate"]]
        eviction_rates_by_tract["GEOID"] = eviction_rates_by_tract["GEOID"].astype(str)

        # Returning final data frame
        evictions_final = filings_df.merge(eviction_rates_by_tract, how='left', on='GEOID')
        evictions_final["eviction-rate"].fillna(overall_eviction_rate, inplace=True)
        evictions_final["eviction-filings"] = evictions_final["evictions"] / evictions_final["eviction-rate"]
        evictions_final = evictions_final[["GEOID", time_group, "eviction-filings", "evictions", "eviction-rate"]]

        # Filtering to desired county's GEOIDs only
        # evictions_final = evictions_final[evictions_final.GEOID.str.contains('^12057')]
    return evictions_final

def load_census_data(data_path):
    """Load the ACS data and generate relevant columns.

    Parameters
    ----------
    data_path : str
        Path to ACS data in CSV format

    Returns
    -------
    pandas df
        Dataframe containing ACS data
    """
    census_cols = {
        "DP03_0051E": "total-households",
        "DP04_0047E": "total-renter-occupied-households",
        "DP04_0046E": "total-owner-occupied-households",
        "S2506_C01_001E": "total-owner-occupied-households-mortgage",
        "B25064_001E": "median-gross-rent",
        "DP03_0062E": "median-household-income",
        "B25077_001E": "median-property-value",
        "S2506_C01_039E": "median-monthly-housing-cost",
        "S2502_C01_002E": "pct-white",
        "S2502_C01_003E": "pct-af-am",
        "S2502_C01_009E": "pct-hispanic",
        "S2502_C01_004E": "pct-am-indian",
        "S2502_C01_005E": "pct-asian",
        "S2502_C01_006E": "pct-nh-pi",
        "S2502_C01_008E": "pct-multiple",
        "S2502_C01_007E": "pct-other",
        "DP03_0119E": "pct-below-poverty-level",
        "DP03_0099E": "without-health-insurance",
        "DP03_0096E": "with-health-insurance",
        "DP02_0003E": "households-children",
        "DP02_0009E": "single-parent-household",
        "DP02_0012E": "older-adult-alone",
        "DP02_0058E": "level-of-education",
        "DP02_0095E": "immigrant-status",
        "DP02_0112E": "english-fluency",
        "DP03_0019E": "drive-to-work",
        "DP03_0021E": "public-transport-to-work",
        "DP04_0003E": "vacant-properties",
        "DP04_0014E": "live-in-mobile-home",
        "B25035_001E": "median-year-structure-built",
    }

    census_df = pd.read_csv(data_path, dtype={"GEOID": str})[
        ["GEOID"] + list(census_cols.keys())
    ].rename(columns=census_cols)

    census_df["pct-renter-occupied"] = (
        census_df["total-renter-occupied-households"] / census_df["total-households"]
    ) * 100
    census_df["pct-owner-occupied"] = (
        census_df["total-owner-occupied-households"] / census_df["total-households"]
    ) * 100
    census_df["pct-owner-occupied-mortgage"] = (
        census_df["total-owner-occupied-households-mortgage"]
        / census_df["total-households"]
    ) * 100
    census_df["pct-owner-occupied-without-mortgage"] = (
        (
            census_df["total-owner-occupied-households"]
            - census_df["total-owner-occupied-households-mortgage"]
        )
        / census_df["total-households"]
        * 100
    )
    census_df["median-house-age"] = (
        datetime.datetime.now().year - census_df["median-year-structure-built"]
    )
    census_df["pct-non-white"] = 100 - census_df["pct-white"]
    census_df["pct-without-health-insurance"] = (
        census_df["without-health-insurance"]
        / (census_df["without-health-insurance"] + census_df["with-health-insurance"])
        * 100
    )
    return census_df.drop(
        [
            "without-health-insurance",
            "with-health-insurance",
            "median-year-structure-built",
        ],
        axis=1,
    )

def get_totals_across_years(df, tract_col, data_col):
    """Get the sum and average of a datapoint across all years for a given geography.

    Parameters
    ----------
    df : pandas df
        Dataframe where each row contains a datapoint by year by geography, e.g. columns =
        ['year', 'tract', 'num_evictions']
    tract_col : str
        Name of the column that contains the tract/geography identifier
    data_col : str
        Name of the column that contains the data point to be aggregated

    Returns
    -------
     : pandas df
        Dataframe containing the sum and average of the data point across all years for each
        geography
    """
    return df.groupby(tract_col)[data_col].agg(["sum", "mean"]).reset_index()

def create_year_cols_from_df(df, data_cols, data_col_prefix_mapper, year_col, geo_col):
    """Generate the year columns from the dataframe containing yearly data in separate rows. The
    resulting dataframe will be reduced to only one row for each geography, with the yearly data
    included in separate columns with the suffix of that year. For example, "evictions" will become
    "evictions-2014".

    Parameters
    ----------
    df : pandas df
        Dataframe with yearly data, with columns containing [year, geo_col, data_col]
    data_cols : list of str
        List of data column names to retain
    data_col_prefix_mapper : dict
        Dictionary mapping original data columns to new prefixes, e.g. {'evictions': 'total-evictions'}
    year_col : str
        Name of the column containing the year information
    geo_col : str
        Name of the geographic identifier column. This will be used as the primary key

    Returns
    -------
    df
        Dataframe with the geo_col column and len(data_cols) * len(years) columns (one column per
        year per data point)
    """
    # Get the unique list of years that are present in the dataframe
    years_list = set(df[year_col].tolist())
    year_dfs = []

    # For each year, generate a dataframe containing only that year's information. Add the year to
    # the end of the data column name and rename as necessary
    for year in years_list:
        year_df = df[df[year_col] == year][[geo_col] + data_cols].rename(
            columns={
                data_col: f"{data_col_prefix_mapper.get(data_col, data_col)}-{str(year)}"
                for data_col in data_cols
            }
        )
        year_dfs.append(year_df)

    # Combine all the year dataframes
    # result = reduce(lambda left, right: pd.merge(left, right, on=geo_col), year_dfs)
    result = reduce(lambda left, right: pd.merge(left, right, on=geo_col, how="outer"), year_dfs)
    result = result.fillna(0)
    return result

def build_rename_mapper_from_df(df, chunk_name_mapper):
    """Build a dictionary where the keys are the current column names and the values are new
    column names based on an input dictionary that contains substrings to rename. For example,
    if the dataframe contains the columns 'evictions-2016' and 'evictions-2017', and the
    chunk_name_mapper contains {'evictions': 'total-evictions'}, this function would generate a new
    dictionary containing {'evictions-2016': 'total-evictions-2016', 'evictions-2017',
    'total-evictions-2017'}. This allows us to generate a dictionary to rename columns that contain
    yearly data without having to manually specify the year.

    Parameters
    ----------
    df : pandas dataframe
        Dataframe containing columns to rename
    chunk_name_mapper : dict of str
        Dict mapping the old substring to the new substring, e.g. {'evictions': 'total-evictions'}

    Returns
    -------
    dict of str
        Dict mapping the old column names to new column names
    """
    rename_mapper = {}
    for column_name_chunk in chunk_name_mapper:
        for c in df.columns:
            if column_name_chunk in c:
                rename_mapper[c] = c.replace(
                    column_name_chunk, chunk_name_mapper[column_name_chunk]
                )
    return rename_mapper

def add_yearly_rates_to_merged_df(
    merged_df, counts_to_rate_name_mapper, rate_denom_col
):
    """Add the yearly rates columns to the merged dataframe by dividing the yearly counts column by
    the rate denominator column.

    Parameters
    ----------
    merged_df : pandas df
        Dataframe containing the year counts and rate denominator columns
    counts_to_rate_name_mapper : dict
        Dictionary that maps the count column name to the rate column name, e.g.
        {'eviction-filings': 'eviction-filing-rate'}
    rate_denom_col : str
        Column containing the data to use as the denominator

    Returns
    -------
    pandas df
        Dataframe containing the yearly rates
    """
    rename_mapper = build_rename_mapper_from_df(merged_df, counts_to_rate_name_mapper)
    rates_df = (
        merged_df[rename_mapper.keys()]
        .div(merged_df[rate_denom_col], axis=0)
        .multiply(100)
        .rename(columns=rename_mapper)
    )
    return pd.concat([merged_df, rates_df,], axis=1,)

def get_counts_by_tract(mf_with_tract, output_col_name):
    """Return a dataframe containing the the mortgage foreclosure counts by Census Tract by year.

    Parameters
    ----------
    mf_with_parcels : pandas df
        Mortgage foreclosure data, where each row represents a foreclosed property and its parcel
        ID

    Returns
    -------
    pandas df
        Dataframe with columns ['year', 'GEOID', 'num_mortgage_foreclosures']
    """

    # Get the counts by census tract by year
    counts_by_ct_by_year = mf_with_tract.groupby(["year", "GEOID"]).size().to_frame('num').reset_index()

    # Reset index to add year column & clean up
    mf_counts_by_ct = (
        counts_by_ct_by_year
        .rename(columns={"num": output_col_name},) # "num_mortgage_foreclosures" or "lien-foreclosures"
    )
    return mf_counts_by_ct

def get_counts_by_month(df, month_col, counts_col_name):
    """Get the count of rows by a given column and rename the counts column.

    Parameters
    ----------
    df : pandas df
        Dataframe containing rows to count and the month column to group by
    month_col : str
        Name of the column containing the month; will be grouped by this column
    counts_col_name : str
        Name to give the column with the counts by month

    Returns
    -------
    pandas df
        Dataframe with columns = ['month', counts_col_name]
    """
    return (
        df.groupby(month_col)
        .count()[df.columns[0]]
        .reset_index()
        .rename(columns={df.columns[0]: counts_col_name})
    )

def generate_time_series_df(eviction_df):
    """Generate the dataframe containing the count of housing loss events by month. Only includes
    mortgage and tax foreclosures since we do not have the evictions by month.

    Parameters
    ----------
    mortgage_df : pandas df
        Dataframe containing the mortgage data; must include "month" column
    tax_df : pandas df
        Dataframe containing the tax data; must include "month" column

    Returns
    -------
    pandas df
        Dataframe containing the mortgage and tax foreclosures counts by month,
        columns = ['month', 'total-foreclosures', 'total-lien-foreclosures']
    """
    evictions_by_month = get_counts_by_month(eviction_df, "month", "total-eviction-filings")
    timeseries_df = evictions_by_month
    return timeseries_df

def reformat_tract_code(tract: str, state_code: str, county_code: str) -> str:
    """Helper function to return GEOIDs compatible with those in deep-dive data files.

    Parameters
    ----------
    tract : A string
        An unformatted tract code, e.g. '53.38'
    state_code : A string
        The FIPS code for the state that we want to append.
    county_code : A string
        The FIPS code for the county that we want to append.

    Returns
    -------
    str
    """

    # If the tract code contains a period, remove it, then prepend zeroes until length is 6
    if "." in tract:
        tract = tract.replace(".", "")
        num_zeroes = 6 - len(tract)
        tract = ("0" * num_zeroes) + tract
    # Else prepend zeroes until the length is 4, then add 2 zeroes to the end
    else:
        num_zeroes = 4 - len(tract)
        tract = ("0" * num_zeroes) + tract + "00"

    # Prepend state and county FIPS codes
    geoid = state_code + county_code + tract

    return geoid

@click.command()
@click.option(
    "--config_path",
    type=click.File("r"),
    help="Path to yaml containing config parameters.",
)
def main(config_path):
    """Process raw Marion, IN housing loss data and write timeseries (counts by month) data and
    aggregate data (by census tract) to CSV.

    Parameters
    ----------
    config_path : str
        Path to the yaml config file containing the input and output file paths
    """
    logging.info("Loading raw data files. This may take a few minutes.")
    config = yaml.load(config_path, yaml.SafeLoader)
    min_year = config["min_year"]
    max_year = config["max_year"]

    # The Eviction Lab data came in two files (2014-2016 and 2017-2018) with different formats, so
    # we need to read them in separately and merge them later.
    eviction_df = load_evictions_data(
        config["eviction_data_path"],
        "EXECUTED_DATE",
        min_year,
        max_year,
        create_geoid = True
    )
    census_df = load_census_data(config["acs_data_path"])

    # Generate the time series data and save output
    logging.info("Generating time series data (monthly counts of housing loss events).")

    # Calculating eviction totals separately because number of evictions does not always equal number of rows in the df
    evictions_monthly = aggregate_evictions_using_rate_estimates(
        eviction_df,
        config["path_to_eviction_filing_rates"],
        estimate_var = "filings",
        time_group = "month"
    )
    evictions_monthly = evictions_monthly[["eviction-filings", "month"]].groupby("month").sum().reset_index()

    # Creating timeseries df and merging on separate eviction counts
    timeseries_df = generate_time_series_df(eviction_df)
    timeseries_df = timeseries_df.merge(evictions_monthly, how="outer")
    timeseries_df.drop_duplicates().to_csv(
        config["timeseries_output_csv_path"], index=False
    )
    logging.info(
        "Output timeseries CSV saved to %s." % config["timeseries_output_csv_path"]
    )

    # Process evictions data--get totals/rates across the analysis period & totals/rates
    # by year
    eviction_df = aggregate_evictions_using_rate_estimates(
        eviction_df,
        config["path_to_eviction_filing_rates"],
        estimate_var = "filings",
        time_group = "year"
    )
    eviction_years_df = create_year_cols_from_df(
        eviction_df,
        ["evictions", "eviction-filings", "eviction-rate"], # "eviction-filing-rate"
        {"evictions": "total-evictions"},
        "year",
        "GEOID",
    )

    eviction_totals = get_totals_across_years(eviction_df, "GEOID", "evictions").rename(
        columns={"sum": "total-evictions", "mean": "avg-evictions"}
    )
    eviction_filing_totals = get_totals_across_years(
        eviction_df, "GEOID", "eviction-filings",
    ).rename(columns={"sum": "total-eviction-filings", "mean": "avg-eviction-filings"})

    # Join evictions, mortgage, tax, and ACS data together into a single dataframe
    merged = (
        census_df
        .merge(eviction_totals, on="GEOID", how="left")
        .merge(eviction_df[["GEOID"]].dropna().drop_duplicates(),on="GEOID",how="left",)
        .merge(eviction_years_df, on="GEOID", how="left")
        )

    merged["overall-city-eviction-rate"] = ((np.sum(merged["avg-evictions"][np.isfinite(merged["avg-evictions"])]))/sum(merged["total-renter-occupied-households"]))*100

    merged["avg-eviction-rate"] = (
        merged["avg-evictions"] / merged["total-renter-occupied-households"]
    ) * 100

    merged["ratio-to-mean-eviction-rate"] = (
        merged["avg-eviction-rate"] / merged["overall-city-eviction-rate"]
    )

    # Add geographic identifier columns
    merged = merged.rename(columns={"GEOID": "census_tract_GEOID"})
    merged["county_GEOID"] = merged["census_tract_GEOID"].apply(lambda x: x[:5])
    merged.loc[merged["county_GEOID"] == "36005", "county"] = "Bronx"
    merged.loc[merged["county_GEOID"] == "36047", "county"] = "Brooklyn"
    merged.loc[merged["county_GEOID"] == "36061", "county"] = "Manhattan"
    merged.loc[merged["county_GEOID"] == "36081", "county"] = "Queens"
    merged.loc[merged["county_GEOID"] == "36085", "county"] = "Staten Island"
    merged["state"] = "New York"

    # Write main output file to CSV
    merged.drop_duplicates().to_csv(config["output_csv_path"], index=False)
    logging.info("Output CSV saved to %s." % config["output_csv_path"])

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO, format="%(asctime)s - %(levelname)s: %(message)s"
    )
    main()


#!/usr/bin/env python

import socks
import socket
import requests
import xml.etree.ElementTree as ET

api_key = [api_key]

socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 8080)
socket.socket = socks.socksocket

#barcodes = ("/alma/bin/clare_barcodes.txt")
#barcodeFile = open(barcodes, "r")
#for line in barcodeFile:
#    line = line.replace("\n", "")

#line = '000014046842'
def lsc_scanin(line):
    barcode = line

    print(barcode)
# send a get using the item barcode to get the rest of the item info
    payload = { 'item_barcode' : barcode, 'apikey' : api_key}

    r = requests.get('https://api-na.hosted.exlibrisgroup.com/almaws/v1/items?', params=payload)

# get the mms_id, holding_id, and item_pid out with elementTree
    root = ET.fromstring(r.text)

#feed them to the request payload
    mmsid = root[0][0].text
    holdingid = root[1][0].text
    pid = root[2][0].text

    payload2 = {'op' : 'scan', 'library' : 'LSC', 'circ_desk' : 'DEFAULT_CIRC_DESK', 'apikey' : api_key}

# send a post to the scan-in url

    r2 = requests.post('https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/' + mmsid + '/holdings/' + holdingid + '/items/' + pid + '?', params=payload2)

    root2 = ET.fromstring(r2.text)
    print(r2.status_code)
    print(root2[2][4].attrib)

# for each barcode in incoming list, call the scanin function
barcodes = ("/alma/bin/clare_barcodes.txt")
barcodeFile = open(barcodes, "r")
for line in barcodeFile:
    line = line.replace("\n", "")
    lsc_scanin(line)


from __future__ import annotations

import json
import math
import shutil
import subprocess

import pytest

import scitbx.matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import Crystal, Experiment, ExperimentList, Scan
from dxtbx.serialize import load

from dials.algorithms.symmetry.cosym._generate_test_data import (
    generate_experiments_reflections,
)
from dials.array_family import flex
from dials.command_line import symmetry
from dials.command_line.symmetry import (
    apply_change_of_basis_ops,
    change_of_basis_ops_to_minimum_cell,
    eliminate_sys_absent,
    get_subset_for_symmetry,
    median_unit_cell,
)
from dials.util.exclude_images import exclude_image_ranges_from_scans
from dials.util.multi_dataset_handling import assign_unique_identifiers
from dials.util.phil import parse

def test_symmetry_laue_only(dials_data, tmp_path):
    """Simple test to check that dials.symmetry completes"""

    lcyst_data = dials_data("l_cysteine_dials_output", pathlib=True)
    result = subprocess.run(
        [
            shutil.which("dials.symmetry"),
            lcyst_data / "20_integrated_experiments.json",
            lcyst_data / "20_integrated.pickle",
            lcyst_data / "25_integrated_experiments.json",
            lcyst_data / "25_integrated.pickle",
            "systematic_absences.check=False",
        ],
        cwd=tmp_path,
        capture_output=True,
    )
    assert not result.returncode and not result.stderr
    assert tmp_path.joinpath("symmetrized.refl").is_file()
    assert tmp_path.joinpath("symmetrized.expt").is_file()
    exps = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(exps[0].crystal.get_space_group().info()) == "P 2 2 2"
    joint_reflections = flex.reflection_table.from_file(tmp_path / "symmetrized.refl")
    # check that there are 2 unique id and imageset_ids, and that these
    # correctly correspond to each experiment
    assert len(set(joint_reflections["id"])) == 2
    assert len(set(joint_reflections["imageset_id"])) == 2
    for id_ in range(2):
        sel = joint_reflections["id"] == id_
        assert set(joint_reflections["imageset_id"].select(sel)) == {id_}

def test_symmetry_basis_changes_for_C2(tmp_path):
    """Test the correctness of change of basis operations in dials.symmetry

    Supply the unit cell of beta-lactamase, which triggers a change of
    basis from input to minimum during symmetry analysis."""
    unit_cell = (53.173, 61.245, 69.292, 90.0, 93.04675, 90.0)
    space_group = sgtbx.space_group_info("C 2").group()
    experiments, reflections, _ = generate_experiments_reflections(
        space_group=space_group,
        unit_cell=unit_cell,
        sample_size=1,
        map_to_minimum=False,
    )
    experiments.as_json(tmp_path / "tmp.expt")
    joint_table = flex.reflection_table()
    for r in reflections:
        joint_table.extend(r)
    joint_table.as_file(tmp_path / "tmp.refl")

    result = subprocess.run(
        [
            shutil.which("dials.symmetry"),
            tmp_path / "tmp.expt",
            tmp_path / "tmp.refl",
            "json=symmetry.json",
        ],
        cwd=tmp_path,
        capture_output=True,
    )
    assert not result.returncode and not result.stderr
    assert tmp_path.joinpath("symmetrized.refl").is_file()
    symmetrized_expt_file = tmp_path / "symmetrized.expt"
    assert symmetrized_expt_file.is_file()

    expts = load.experiment_list(symmetrized_expt_file, check_format=False)
    for v, expected in zip(expts[0].crystal.get_unit_cell().parameters(), unit_cell):
        assert v == pytest.approx(expected)

    # Using the change of basis ops from the json output we should be able to
    # reindex the input experiments to match the output experiments
    with tmp_path.joinpath("symmetry.json").open() as f:
        d = json.load(f)
    cs = experiments[0].crystal.get_crystal_symmetry()
    cb_op_inp_min = sgtbx.change_of_basis_op(str(d["cb_op_inp_min"][0]))
    cb_op_min_best = sgtbx.change_of_basis_op(str(d["subgroup_scores"][0]["cb_op"]))
    assert cs.change_basis(cb_op_min_best * cb_op_inp_min).is_similar_symmetry(
        expts[0].crystal.get_crystal_symmetry()
    )

@pytest.mark.parametrize("option", ["", "exclude_images=0:1500:1800"])
def test_symmetry_with_absences(dials_data, tmp_path, option):
    """Simple test to check that dials.symmetry, with absences, completes"""

    lcyst_data = dials_data("l_cysteine_dials_output", pathlib=True)
    cmd = [
        shutil.which("dials.symmetry"),
        lcyst_data / "20_integrated_experiments.json",
        lcyst_data / "20_integrated.pickle",
        lcyst_data / "25_integrated_experiments.json",
        lcyst_data / "25_integrated.pickle",
    ]
    if option:
        cmd.append(option)

    result = subprocess.run(cmd, cwd=tmp_path, capture_output=True)
    assert not result.returncode and not result.stderr
    assert (tmp_path / "symmetrized.refl").is_file()
    assert (tmp_path / "symmetrized.expt").is_file()
    expts = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(expts[0].crystal.get_space_group().info()) == "P 21 21 21"

def test_symmetry_with_laue_group_override(dials_data, tmp_path):
    """Simple test to check that dials.symmetry, with overridden laue group, completes"""

    lcyst_data = dials_data("l_cysteine_dials_output", pathlib=True)
    result = subprocess.run(
        [
            shutil.which("dials.symmetry"),
            "laue_group=P121",
            "change_of_basis_op=-b,-a,-c",
            lcyst_data / "20_integrated_experiments.json",
            lcyst_data / "20_integrated.pickle",
            lcyst_data / "25_integrated_experiments.json",
            lcyst_data / "25_integrated.pickle",
        ],
        cwd=tmp_path,
        capture_output=True,
    )
    assert not result.returncode and not result.stderr
    assert tmp_path.joinpath("symmetrized.refl").is_file()
    assert tmp_path.joinpath("symmetrized.expt").is_file()
    expts = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(expts[0].crystal.get_space_group().info()) == "P 1 21 1"
    # Verify that the unit cell has been reindexed correctly
    assert expts[0].crystal.get_unit_cell().parameters() == pytest.approx(
        (8.21578444269, 5.4815363434, 12.1457047712, 90.0, 90.0, 90.0)
    )

@pytest.mark.parametrize("method", ["direct", "fourier"])
def test_symmetry_absences_only(dials_data, tmp_path, method):
    """Test the command line script with real data. Proteinase K in P41"""
    location = dials_data("vmxi_proteinase_k_sweeps", pathlib=True)

    command = [
        shutil.which("dials.symmetry"),
        "laue_group=None",
        location / "experiments_0.json",
        location / "reflections_0.pickle",
        f"method={method}",
    ]
    result = subprocess.run(command, cwd=tmp_path, capture_output=True)
    assert not result.returncode and not result.stderr
    assert tmp_path.joinpath("dials.symmetry.html").is_file()
    assert tmp_path.joinpath("symmetrized.expt").is_file()
    exps = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(exps[0].crystal.get_space_group().info()) == "P 41"

    # Now try with a d_min
    command += ["d_min=4.0"]
    result = subprocess.run(command, cwd=tmp_path, capture_output=True)
    assert not result.returncode and not result.stderr
    exps = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(exps[0].crystal.get_space_group().info()) == "P 41"

def test_map_to_minimum_cell():
    # Input and expected output
    input_ucs = [
        (39.7413, 183.767, 140.649, 90, 90, 90),
        (40.16, 142.899, 92.4167, 90, 102.48, 90),
        (180.613, 40.1558, 142.737, 90, 90.0174, 90),
    ]
    input_sgs = ["C 2 2 21", "P 1 2 1", "C 1 2 1"]
    input_hkl = [
        [(1, -75, -71), (1, -73, -70), (1, -71, -69)],
        [(14, -37, -36), (-2, -35, -46), (-3, -34, -47)],
        [(-31, -5, -3), (-25, -3, -3), (-42, -8, -2)],
    ]
    expected_ucs = [
        (39.7413, 94.00755450320204, 140.649, 90.0, 90.0, 77.79717980856927),
        (40.16, 92.46399390642911, 142.899, 90.0, 90.0, 77.3882749092846),
        (
            40.1558,
            92.51154528306184,
            142.73699999999997,
            89.9830147351441,
            90.0,
            77.46527404307477,
        ),
    ]
    expected_output_hkl = [
        [(-1, 37, -71), (-1, 36, -70), (-1, 35, -69)],
        [(-14, 22, 37), (2, 48, 35), (3, 50, 34)],
        [(-5, 13, -3), (-3, 11, -3), (-8, 17, -2)],
    ]

    # Setup the input experiments and reflection tables
    expts = ExperimentList()
    reflections = []
    for uc, sg, hkl in zip(input_ucs, input_sgs, input_hkl):
        uc = uctbx.unit_cell(uc)
        sg = sgtbx.space_group_info(sg).group()
        B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
        expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))
        refl = flex.reflection_table()
        refl["miller_index"] = flex.miller_index(hkl)
        reflections.append(refl)

    # Actually run the method we are testing
    cb_ops = change_of_basis_ops_to_minimum_cell(
        expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
    )
    cb_ops_as_xyz = [cb_op.as_xyz() for cb_op in cb_ops]
    # Actual cb_ops are machine dependent (sigh)
    assert cb_ops_as_xyz == [
        "-x+y,-2*y,z",
        "-x+z,-z,-y",
        "x+y,-2*x,z",
    ] or cb_ops_as_xyz == ["x-y,2*y,z", "x-z,z,-y", "-x-y,2*x,z"]

    expts_min, reflections = apply_change_of_basis_ops(expts, reflections, cb_ops)
    # Verify that the unit cells have been transformed as expected
    for expt, uc in zip(expts, expected_ucs):
        assert expt.crystal.get_unit_cell().parameters() == pytest.approx(uc, abs=4e-2)

    # Space group should be set to P1
    assert [expt.crystal.get_space_group().type().number() for expt in expts_min] == [
        1,
        1,
        1,
    ]

    # Verify that the reflections have been reindexed as expected
    # Because the exact choice of minimum cell can be platform-dependent,
    # compare the magnitude, but not the sign of the output hkl values
    for refl, expected_hkl in zip(reflections, expected_output_hkl):
        for hkl, e_hkl in zip(refl["miller_index"], expected_hkl):
            assert [abs(h) for h in hkl] == [abs(eh) for eh in e_hkl]

def _make_input_for_exclude_tests(exclude_images=True):
    """Generate input data, that upon exclusion should leave only the first
    reflection."""
    if exclude_images:
        exclude_images = [["0:360:720"], ["1:360:720"]]
    expt1 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, 1.0)))
    expt2 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, -1.0)))
    refls1 = flex.reflection_table()
    refls2 = flex.reflection_table()
    refls1["xyzobs.mm.value"] = flex.vec3_double(
        [(0.0, 0.0, 10.0 * math.pi / 180.0), (0.0, 0.0, 370.0 * math.pi / 180.0)]
    )
    refls1["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)])
    refls1["i"] = flex.int([0, 1])
    refls2["xyzobs.mm.value"] = flex.vec3_double(
        [(0.0, 0.0, -10.0 * math.pi / 180.0), (0.0, 0.0, -370.0 * math.pi / 180.0)]
    )
    refls2["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)])
    refls2["i"] = flex.int([0, 1])
    expts = ExperimentList([expt1, expt2])
    tables = [refls1, refls2]
    expts, tables = assign_unique_identifiers(expts, tables)
    return exclude_images, expts, tables

def test_get_subset_for_symmetry_image_range():
    """Test that first 360 degrees are selected from each sweep with an exclude
    images command."""
    exclude_images, expts, tables = _make_input_for_exclude_tests(exclude_images=True)
    refls = get_subset_for_symmetry(expts, tables, exclude_images)
    assert refls[0]["i"].all_eq(0)
    assert refls[1]["i"].all_eq(0)

def test_get_subset_for_symmetry_prior_image_range():
    """Test that first 360 degrees are selected from each sweep with an exclude
    images command."""
    exclude_images, expts, tables = _make_input_for_exclude_tests(exclude_images=True)

    # Explicitly exclude a different image range
    expts = exclude_image_ranges_from_scans(tables, expts, [["0:1:360"], ["1:1:360"]])
    refls = get_subset_for_symmetry(expts, tables)
    assert refls[0]["i"].all_eq(1)
    assert refls[1]["i"].all_eq(1)

    # Exclude_images should be cumulative, i.e. the range exclude above should
    # be excluded in addition to the new range provided explicitly to the
    # function
    refls = get_subset_for_symmetry(expts, tables, exclude_images)
    assert len(refls[0]) == 0
    assert len(refls[1]) == 0

def test_get_subset_for_symmetry_first_360():
    """Test that first 360 degrees are selected from each sweep"""
    exclude_images, expts, tables = _make_input_for_exclude_tests(exclude_images=False)
    refls = get_subset_for_symmetry(expts, tables, exclude_images)
    assert refls[0]["i"].all_eq(0)
    assert refls[1]["i"].all_eq(0)

def test_change_of_basis_ops_to_minimum_cell_1037(mocker):
    # See https://github.com/dials/dials/issues/1037

    input_ucs = [
        (
            4.805202948916906,
            12.808064769657364,
            16.544899201125446,
            106.45808502003258,
            90.0065567098825,
            100.77735674275475,
        ),
        (
            4.808011343212577,
            12.821894835790472,
            16.557339561965573,
            106.48431244651402,
            90.0252848479048,
            100.77252933676507,
        ),
        (
            4.8096632137789985,
            12.815648858527567,
            16.55931712239122,
            106.48990701341536,
            90.01703141314147,
            100.80397887485773,
        ),
        (
            4.807294085194974,
            12.822386757910516,
            16.560411742466663,
            106.43185845358086,
            90.02067929544215,
            100.79522302759383,
        ),
    ]

    # Setup the input experiments and reflection tables
    expts = ExperimentList()
    for uc in input_ucs:
        uc = uctbx.unit_cell(uc)
        sg = sgtbx.space_group_info("P1").group()
        B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
        expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))

    # We want to spy on the return value of this function
    mocker.spy(symmetry, "unit_cells_are_similar_to")

    # Actually run the method we are testing
    cb_ops = change_of_basis_ops_to_minimum_cell(
        expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
    )
    import pytest_mock

    if getattr(pytest_mock, "version", "").startswith("1."):
        assert symmetry.unit_cells_are_similar_to.return_value is True
    else:
        assert symmetry.unit_cells_are_similar_to.spy_return is True
    cb_ops_as_xyz = [cb_op.as_xyz() for cb_op in cb_ops]
    assert len(set(cb_ops_as_xyz)) == 1
    # Actual cb_ops are machine dependent (sigh)
    assert cb_ops_as_xyz[0] in ("x,y,z", "-x,y,-z", "x-y,-y,-z")

def test_change_of_basis_ops_to_minimum_cell_mpro():
    input_ucs = [
        (46.023, 55.001, 64.452, 64.744, 78.659, 89.824),
        (44.747, 53.916, 62.554, 114.985, 99.610, 90.736),
    ]

    # Setup the input experiments and reflection tables
    expts = ExperimentList()
    for uc in input_ucs:
        uc = uctbx.unit_cell(uc)
        sg = sgtbx.space_group_info("P1").group()
        B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
        expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))

    # Actually run the method we are testing
    cb_ops = change_of_basis_ops_to_minimum_cell(
        expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
    )
    expts.change_basis(cb_ops, in_place=True)
    assert symmetry.unit_cells_are_similar_to(
        expts,
        median_unit_cell(expts),
        relative_length_tolerance=0.05,
        absolute_angle_tolerance=2,
    )

from cctbx import crystal

def test_change_of_basis_ops_to_minimum_cell_with_outlier():
    symmetries = [
        crystal.symmetry(unit_cell=uc, space_group="P1")
        for uc in (
            (52.8868, 52.8868, 333.522, 90, 90, 120),
            (52.6503, 53.0292, 333.783, 89.9872, 89.2247, 60.8078),
            (52.9571, 53.0005, 334.255, 90.0493, 90.0042, 119.893),
            (
                54.4465,
                56.5677,
                355.775,
                93.4376,
                90.0999,
                118.256,
            ),  # This is an outlier
            (52.9235, 52.9235, 335.296, 90, 90, 120),
            (53.4531, 53.4531, 322.909, 90, 90, 120),
        )
    ]

    # Setup the input experiments and reflection tables
    expts = ExperimentList()
    for cs in symmetries:
        B = scitbx.matrix.sqr(cs.unit_cell().fractionalization_matrix()).transpose()
        expts.append(
            Experiment(
                crystal=Crystal(B, space_group=cs.space_group(), reciprocal=True)
            )
        )

    # Actually run the method we are testing
    cb_ops = change_of_basis_ops_to_minimum_cell(
        expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
    )
    assert cb_ops.count(None) == 1
    assert cb_ops[3] is None
    expts = ExperimentList([expt for expt, cb_op in zip(expts, cb_ops) if cb_op])
    cb_ops = [cb_op for cb_op in cb_ops if cb_op]

    expts.change_basis(cb_ops, in_place=True)
    assert symmetry.unit_cells_are_similar_to(
        expts,
        median_unit_cell(expts),
        relative_length_tolerance=0.05,
        absolute_angle_tolerance=2,
    )

def test_median_cell():
    unit_cells = [
        uctbx.unit_cell(uc)
        for uc in [
            (10, 11, 11.9, 90, 85, 90),
            (10.1, 11.2, 12, 90, 85.5, 90),
            (10.2, 11.1, 12, 90, 84.7, 90),
        ]
    ]
    expts = ExperimentList()
    for uc in unit_cells:
        sg = sgtbx.space_group_info("P1").group()
        B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
        expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))

    median = median_unit_cell(expts)
    assert median.parameters() == pytest.approx((10.1, 11.1, 12, 90, 85, 90))

def test_eliminate_sys_absent():
    refl = flex.reflection_table()
    refl["miller_index"] = flex.miller_index(
        [(-31, -5, -3), (-25, -3, -3), (0, 1, 0), (-42, -8, -2)]
    )
    sgi = sgtbx.space_group_info("C121")
    uc = sgi.any_compatible_unit_cell(volume=1000)
    B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
    expt = Experiment(crystal=Crystal(B, space_group=sgi.group(), reciprocal=True))
    reflections = eliminate_sys_absent([expt], [refl])
    assert list(reflections[0]["miller_index"]) == [
        (-31, -5, -3),
        (-25, -3, -3),
        (-42, -8, -2),
    ]

def test_few_reflections(dials_data, run_in_tmp_path):
    """
    Test that dials.symmetry does something sensible if given few reflections.

    Use some example integrated data generated from a ten-image 1° sweep.  These
    contain a few dozen integrated reflections.

    Args:
        dials_data: DIALS custom Pytest fixture for access to test data.
        run_in_tmp_path: DIALS custom Pytest fixture to run this test in a temporary
                         directory.
    """
    # Get and use the default parameters for dials.symmetry.
    params = symmetry.phil_scope.fetch(source=parse("")).extract()

    # Use the integrated data from the first ten images of the first sweep.
    data_dir = dials_data("l_cysteine_dials_output", pathlib=True)
    experiments = ExperimentList.from_file(data_dir / "11_integrated.expt")
    reflections = [flex.reflection_table.from_file(data_dir / "11_integrated.refl")]

    # Run dials.symmetry on the above data files.
    symmetry.symmetry(experiments, reflections, params)

def test_x4wide(dials_data, tmp_path):
    """Verify reflections are reindexed before systematic absence analysis.

    Exercise a bug fix in https://github.com/dials/dials/pull/2183

    Expected space group is P 43 21 2 (or its enantiomorphic equivalent, P 41 21 2)
    See also https://www.rcsb.org/structure/3QF8
    """
    x4wide = dials_data("x4wide_processed", pathlib=True)
    result = subprocess.run(
        [
            shutil.which("dials.symmetry"),
            x4wide / "AUTOMATIC_DEFAULT_scaled.expt",
            x4wide / "AUTOMATIC_DEFAULT_scaled.refl",
        ],
        cwd=tmp_path,
    )
    assert not result.returncode and not result.stderr
    assert tmp_path.joinpath("symmetrized.refl").is_file()
    assert tmp_path.joinpath("symmetrized.expt").is_file()
    exps = load.experiment_list(tmp_path / "symmetrized.expt", check_format=False)
    assert str(exps[0].crystal.get_space_group().info()) == "P 41 21 2"


import time
import praw
import config as cfg, rants

class Comments:
    """ Instance and actions for each comment visited. """

    def __init__(self, db=None, r=None, post=None) -> None:
        self.db = db
        self.r = r
        if type(post) is not praw.models.Submission:
            post.refresh()
        self.post = post

    def getParent(self) -> str:
        """ Returns the parent of post. """

        if type(self.post) is praw.models.Submission:
            return(self.post)
        elif type(self.post) is praw.models.Comment:
            return(self.post.parent())

    def markVisited(self) -> None:
        """ Mark a post as visited if not duplicate. """

        id = str(self.post.id)
        if type(self.post) is praw.models.Submission:
            id = "sub " + id

        if id not in cfg.already_visited:
            cfg.already_visited.append(id)

    def alreadyVisited(self) -> bool:
        """ Checks if a post had already been visited. """

        id = str(self.post.id)
        if type(self.post) is praw.models.Submission:
            id = "sub " + id

        if id in cfg.already_visited:
            if cfg.DEBUG: print("Already visited, skipping.")
            return True
        return False

    def postReply(self, reply) -> None:
        """ Add the post to list of visited, post reply. """

        if cfg.DEBUG:
            print("Posting reply to "+ str(self.post.id) +":\n"+ reply)
        elif not cfg.HOSTED: print("X", end="")

        parent = self.getParent()
        if parent.author is None or parent.author == "[deleted]":
            if cfg.DEBUG: print("Post was deleted, skipping.")
            return None
        elif parent.author == cfg.ZENBOT_USERNAME:
            if cfg.DEBUG: print("Reply is to bot, skipping.")
            return None
        elif parent.author == cfg.AUTHOR:
            if cfg.DEBUG: print("Replying to poster instead.")
            parent = self.post

        try:
            parent.reply(reply + cfg.sig)
            time.sleep(cfg.RATELIMIT)
            self.db.writeVisited()
        except praw.exceptions.APIException as err:
            raise cfg.ExitException(err)
        except:
            raise cfg.ExitException("ERROR: Post reply error.")

    def checkComment(self) -> None:
        """ Post replies to random submissions and trigger replies. """

        if cfg.DEBUG: print("comment: "+ format(self.post))
        elif not cfg.HOSTED: print(".", end="", flush=True)

        replies = self.post.replies
        for reply in replies:
            newc = Comments(self.db, self.r, reply)
            newc.checkComment()

        if self.alreadyVisited():
            return
        self.markVisited()

        if (cfg.TRIGGER in self.post.body):
            if (cfg.CMD_KOAN in self.post.body):
                self.postReply(self.db.readRandom(cfg.KOAN_STORE))
            elif (cfg.CMD_RANT in self.post.body):
                self.postReply(rants.Rants(self.db).getTemplate())
            elif (cfg.CMD_HAIKU in self.post.body):
                self.postReply(self.db.readRandom(cfg.HAIKU_STORE))
            elif (cfg.CMD_REPLY in self.post.body):
                self.postReply(cfg.botReply(self.db.readRandom(cfg.REPLY_STORE)))
            cfg.already_visited.append(str(self.post.id))