python source code of aggregate_lending_recall_values

Project: ml-fairness-gym (GitHub Link)

ml-fairness-gym-master
- AUTHORS
- params_test.py
- distributions.py
- LICENSE
- file_util.py
- CONTRIBUTING.md
- distributions_test.py
- test_util.py
- runner_lib_test.py
- run.sh
- params.py
- runner_lib.py
- agents
  - recommenders
    - rnn_cvar_agent.py
    - evaluation.py
    - model_test.py
    - rnn_agent_tests.py
    - rnn_agent.py
    - model.py
    - evaluation_test.py
    - __init__.py
    - batched_movielens_rnn_agent_test.py
    - utils.py
    - batched_movielens_rnn_agent.py
  - random_agents_test.py
  - college_admission_jury.py
  - threshold_policies_test.py
  - allocation_agents_test.py
  - allocation_agents.py
  - classifier_agents.py
  - infectious_disease_agents_test.py
  - threshold_policies.py
  - classifier_agents_test.py
  - oracle_lending_agent.py
  - infectious_disease_agents.py
  - college_admission_jury_test.py
  - oracle_lending_agent_test.py
  - random_agents.py
- experiments
  - acm_fat_2020
    - aggregate_lending_recall_values.py
    - college_admission_main.py
    - attention_allocation_experiment_fat_main.py
    - lending_experiments_main.py
  - movielens_recs_main.py
  - movielens_recs_test.py
  - infectious_disease_test.py
  - attention_allocation_experiment.py
  - infectious_disease_util.py
  - college_admission_util_test.py
  - lending_demo.py
  - movielens_recs.py
  - config
    - college_admission_config.gin
    - example_config.gin
  - lending_test.py
  - lending_plots.py
  - lending_plots_test.py
  - lending.py
  - infectious_disease_rl.py
  - infectious_disease_rl_test.py
  - infectious_disease.py
  - README.md
  - college_admission.py
  - college_admission_util.py
  - attention_allocation_experiment_test.py
  - attention_allocation_experiment_plotting.py
  - college_admission_test.py
- run_util.py
- rewards_test.py
- core_test.py
- rewards.py
- file_util_test.py
- README.md
- papers
- tests.sh
- runner.py
- requirements.txt
- core.py
- spaces
  - multinomial.py
  - multinomial_test.py
  - graph.py
  - multi_discrete_with_none_test.py
  - graph_test.py
  - multi_discrete_with_none.py
  - batch_test.py
  - batch.py
- metrics
  - error_metrics.py
  - infectious_disease_metrics_test.py
  - distribution_comparison_metrics_test.py
  - distribution_comparison_metrics.py
  - lending_metrics_test.py
  - infectious_disease_metrics.py
  - value_tracking_metrics_test.py
  - lending_metrics.py
  - value_tracking_metrics.py
  - error_metrics_test.py
- environments
  - recommenders
    - recsim_wrapper.py
    - download_movielens.py
    - movie_lens_dynamic.py
    - test_data
      - movies.csv
      - users.csv
      - ratings.csv
      - embeddings.json
    - movie_lens_utils.py
    - recsim_wrapper_test.py
    - restaurant_toy_recsim.py
    - README.md
    - movie_lens_dynamic_test.py
    - restaurant_toy_recsim_test.py
    - recsim_samplers.py
  - infectious_disease_test.py
  - template_test.py
  - attention_allocation.py
  - template.py
  - lending_test.py
  - lending.py
  - infectious_disease.py
  - lending_params.py
  - README.md
  - college_admission.py
  - attention_allocation_test.py
  - college_admission_test.py
- docs
  - quickstart.md
  - img
  - using_runner.md
  - installation.md
  - FAQ.md
  - using_ml_fairness_gym.md

# coding=utf-8
# Copyright 2020 The ML Fairness Gym Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Code to aggregate multiple runs for cumulative-recall measurements.

To recreate the recall-gap plot from the lending experiments in the paper,
first run:

for seed in {0..99}
  experiments/acm_fat_2020/lending_experiments_main.py --seed=${seed}
done

Then run:

experiments/acm_fat_2020/aggregate_lending_recall_values.py
"""

from __future__ import absolute_import
from __future__ import division

from __future__ import print_function

import os
from absl import app
from absl import flags
import file_util
from experiments import lending_plots
import numpy as np

FLAGS = flags.FLAGS

flags.DEFINE_string('source_path', '/tmp/fairness_gym/lending_plots',
                    'Root directory where results are stored. Under the root '
                    'directory there should be subdirectories for each random '
                    'seed containing run statistics.')


flags.DEFINE_string('plotting_dir', '/tmp/fairness_gym/lending_plots',
                    'Path to write plots.')

flags.DEFINE_integer('num_trials', 100, 'Number of trials to aggregate.')


def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')

  recall_values = {}
  for setting in ['static', 'dynamic']:
    recall = []
    for seed in range(FLAGS.num_trials):
      with file_util.open(
          os.path.join(FLAGS.source_path, str(seed), 'cumulative_recall_%s.txt')
          % setting, 'r') as infile:
        recall.append(np.loadtxt(infile))
      recall_values[setting] = np.vstack(recall)

  lending_plots.plot_cumulative_recall_differences(
      recall_values,
      path=os.path.join(FLAGS.plotting_dir, 'combined_simpsons_paradox.pdf'))


if __name__ == '__main__':
  app.run(main)