Index

Symbols

$AWS_SECURITY_TOKEN, [1] $AWS_SESSION_TOKEN $HADOOP_*

$mapreduce_map_input_file $PYTHONPATH

_

__init__() (mrjob.hadoop.HadoopJobRunner method) (mrjob.inline.InlineMRJobRunner method) (mrjob.job.MRJob method) (mrjob.local.LocalMRJobRunner method) (mrjob.retry.RetryWrapper method) (mrjob.runner.MRJobRunner method)

A

add() (mrjob.setup.UploadDirManager method) (mrjob.setup.WorkingDirManager method) add_file_arg() (mrjob.job.MRJob method) add_passthru_arg() (mrjob.job.MRJob method) ARCHIVES (mrjob.job.MRJob attribute)

archives() (mrjob.job.MRJob method) AWS_ACCESS_KEY_ID, [1] AWS_SECRET_ACCESS_KEY, [1] AWS_SESSION_TOKEN

B

bunzip2_stream() (in module mrjob.cat) BytesProtocol (class in mrjob.protocol)

BytesValueProtocol (class in mrjob.protocol)

C

can_handle_path() (mrjob.fs.base.Filesystem method) cat() (mrjob.fs.base.Filesystem method) cat_output() (mrjob.runner.MRJobRunner method) cleanup() (mrjob.runner.MRJobRunner method) CLEANUP_CHOICES (in module mrjob.options) cmd_line() (in module mrjob.util) combine_cmds() (in module mrjob.conf) combine_dicts() (in module mrjob.conf) combine_envs() (in module mrjob.conf) combine_jobconfs() (in module mrjob.conf) combine_lists() (in module mrjob.conf) combine_local_envs() (in module mrjob.conf) combine_pairs() (mrjob.job.MRJob method)

combine_path_lists() (in module mrjob.conf) combine_paths() (in module mrjob.conf) combine_values() (in module mrjob.conf) combiner combiner() (mrjob.job.MRJob method) combiner_cmd() (mrjob.job.MRJob method) combiner_final() (mrjob.job.MRJob method) combiner_init() (mrjob.job.MRJob method) combiner_pre_filter() (mrjob.job.MRJob method) configure_args() (mrjob.job.MRJob method) counters() (mrjob.runner.MRJobRunner method) create_bucket() (mrjob.emr.S3Filesystem method)

D

DataprocJobRunner (class in mrjob.dataproc) decompress() (in module mrjob.cat) describe_base_emr_images() (in module mrjob.ami)

DIRS (mrjob.job.MRJob attribute) dirs() (mrjob.job.MRJob method) du() (mrjob.fs.base.Filesystem method)

E

EMRJobRunner (class in mrjob.emr) environment variable $AWS_SECURITY_TOKEN, [1] $AWS_SESSION_TOKEN $HADOOP_* $PYTHONPATH $mapreduce_map_input_file AWS_ACCESS_KEY_ID, [1] AWS_SECRET_ACCESS_KEY, [1] AWS_SESSION_TOKEN MRJOB_CONF, [1], [2], [3] PATH, [1] PYTHONPATH, [1], [2], [3] TZ

exists() (mrjob.fs.base.Filesystem method) expand_path() (in module mrjob.util)

F

file_ext() (in module mrjob.util) FILES (mrjob.job.MRJob attribute) files() (mrjob.job.MRJob method) Filesystem (class in mrjob.fs.base)

find_mrjob_conf() (in module mrjob.conf) fs (mrjob.runner.MRJobRunner attribute) fully_qualify_hdfs_path() (in module mrjob.hadoop)

G

GCSFilesystem (class in mrjob.dataproc) GENERIC_ARGS (in module mrjob.step) get_all_bucket_names() (mrjob.emr.S3Filesystem method) get_bucket() (mrjob.emr.S3Filesystem method) get_cluster_id() (mrjob.emr.EMRJobRunner method) get_hadoop_version() (mrjob.runner.MRJobRunner method)

get_image_version() (mrjob.emr.EMRJobRunner method) get_job_key() (mrjob.runner.MRJobRunner method) get_job_steps() (mrjob.emr.EMRJobRunner method) get_opts() (mrjob.runner.MRJobRunner method) gunzip_stream() (in module mrjob.cat)

H

Hadoop Streaming HADOOP_INPUT_FORMAT (mrjob.job.MRJob attribute) hadoop_input_format() (mrjob.job.MRJob method)

HADOOP_OUTPUT_FORMAT (mrjob.job.MRJob attribute) hadoop_output_format() (mrjob.job.MRJob method) HadoopJobRunner (class in mrjob.hadoop)

I

increment_counter() (mrjob.job.MRJob method) InlineMRJobRunner (class in mrjob.inline) INPUT (in module mrjob.step) input protocol INPUT_PROTOCOL (mrjob.job.MRJob attribute) input_protocol() (mrjob.job.MRJob method)

internal protocol INTERNAL_PROTOCOL (mrjob.job.MRJob attribute) internal_protocol() (mrjob.job.MRJob method) is_s3_uri() (in module mrjob.parse) is_task() (mrjob.job.MRJob method) is_uri() (in module mrjob.parse)

J

JarStep (class in mrjob.step) JOBCONF (mrjob.job.MRJob attribute) jobconf() (mrjob.job.MRJob method) jobconf_from_dict() (in module mrjob.compat)

jobconf_from_env() (in module mrjob.compat) join() (mrjob.fs.base.Filesystem method) JSONProtocol (class in mrjob.protocol) JSONValueProtocol (class in mrjob.protocol)

L

LIBJARS (mrjob.job.MRJob attribute) libjars() (mrjob.job.MRJob method) load_args() (mrjob.job.MRJob method) load_opts_from_mrjob_conf() (in module mrjob.conf) load_opts_from_mrjob_confs() (in module mrjob.conf)

LocalMRJobRunner (class in mrjob.local) log_to_null() (in module mrjob.util) log_to_stream() (in module mrjob.util) ls() (mrjob.fs.base.Filesystem method)

M

make_ec2_client() (mrjob.emr.EMRJobRunner method) make_emr_client() (mrjob.emr.EMRJobRunner method) make_iam_client() (mrjob.emr.EMRJobRunner method) make_runner() (mrjob.job.MRJob method) make_s3_client() (mrjob.emr.S3Filesystem method) make_s3_resource() (mrjob.emr.S3Filesystem method) map_pairs() (mrjob.job.MRJob method) map_version() (in module mrjob.compat) mapper mapper() (mrjob.job.MRJob method) mapper_cmd() (mrjob.job.MRJob method) mapper_final() (mrjob.job.MRJob method) mapper_init() (mrjob.job.MRJob method) mapper_pre_filter() (mrjob.job.MRJob method) mapper_raw() (mrjob.job.MRJob method) md5sum() (mrjob.fs.base.Filesystem method) mkdir() (mrjob.fs.base.Filesystem method) mr_job_script() (mrjob.job.MRJob class method) MRJob (class in mrjob.job) mrjob.ami (module) mrjob.cat (module) mrjob.compat (module) mrjob.conf (module) mrjob.dataproc (module) mrjob.emr (module)

mrjob.fs.base (module) mrjob.hadoop (module) mrjob.inline (module) mrjob.job (module) mrjob.local (module) mrjob.parse (module) mrjob.protocol (module) mrjob.retry (module) mrjob.runner (module) mrjob.setup (module) mrjob.spark.runner (module) mrjob.step (module) mrjob.tools.diagnose (module) mrjob.tools.emr.audit_usage (module) mrjob.tools.emr.create_cluster (module) mrjob.tools.emr.mrboss (module) mrjob.tools.emr.report_long_jobs (module) mrjob.tools.emr.s3_tmpwatch (module) mrjob.tools.emr.terminate_cluster (module) mrjob.tools.emr.terminate_idle_clusters (module) mrjob.tools.spark_submit (module) mrjob.util (module) MRJOB_CONF, [1], [2], [3] MRJobRunner (class in mrjob.runner) MRStep (class in mrjob.step)

N

name() (mrjob.setup.WorkingDirManager method) name_to_path() (mrjob.setup.WorkingDirManager method)

name_uniquely() (in module mrjob.setup)

O

OUTPUT (in module mrjob.step) output protocol

OUTPUT_PROTOCOL (mrjob.job.MRJob attribute) output_protocol() (mrjob.job.MRJob method)

P

parse_legacy_hash_path() (in module mrjob.setup) parse_mr_job_stderr() (in module mrjob.parse) parse_output() (mrjob.job.MRJob method) parse_s3_uri() (in module mrjob.parse) parse_setup_cmd() (in module mrjob.setup) PARTITIONER (mrjob.job.MRJob attribute) partitioner() (mrjob.job.MRJob method) pass_arg_through() (mrjob.job.MRJob method) PATH, [1]

path_to_uri() (mrjob.setup.UploadDirManager method) paths() (mrjob.setup.WorkingDirManager method) pick_protocols() (mrjob.job.MRJob method) PickleProtocol (class in mrjob.protocol) PickleValueProtocol (class in mrjob.protocol) protocol put() (mrjob.fs.base.Filesystem method) PYTHONPATH, [1], [2], [3]

R

random_identifier() (in module mrjob.util) RapidJSONProtocol (class in mrjob.protocol) RapidJSONValueProtocol (class in mrjob.protocol) RawProtocol (class in mrjob.protocol) RawValueProtocol (class in mrjob.protocol) reduce_pairs() (mrjob.job.MRJob method) reducer reducer() (mrjob.job.MRJob method) reducer_cmd() (mrjob.job.MRJob method) reducer_final() (mrjob.job.MRJob method) reducer_init() (mrjob.job.MRJob method)

reducer_pre_filter() (mrjob.job.MRJob method) ReprProtocol (class in mrjob.protocol) ReprValueProtocol (class in mrjob.protocol) RetryWrapper (class in mrjob.retry) rm() (mrjob.fs.base.Filesystem method) run() (mrjob.job.MRJob class method) (mrjob.runner.MRJobRunner method) run_combiner() (mrjob.job.MRJob method) run_job() (mrjob.job.MRJob method) run_mapper() (mrjob.job.MRJob method) run_reducer() (mrjob.job.MRJob method)

S

S3Filesystem (class in mrjob.fs.s3) safeeval() (in module mrjob.util) sandbox() (mrjob.job.MRJob method) save_current_environment() (in module mrjob.util) save_cwd() (in module mrjob.util) save_sys_path() (in module mrjob.util) save_sys_std() (in module mrjob.util) set_status() (mrjob.job.MRJob method) shlex_split() (in module mrjob.util) SimpleJSONProtocol (class in mrjob.protocol) SimpleJSONValueProtocol (class in mrjob.protocol)

SORT_VALUES (mrjob.job.MRJob attribute) spark() (mrjob.job.MRJob method) SparkJarStep (class in mrjob.step) SparkMRJobRunner (class in mrjob.spark.runner) SparkScriptStep (class in mrjob.step) SparkStep (class in mrjob.step) StandardJSONProtocol (class in mrjob.protocol) StandardJSONValueProtocol (class in mrjob.protocol) step steps() (mrjob.job.MRJob method) strip_microseconds() (in module mrjob.util)

T

TextProtocol (class in mrjob.protocol) TextValueProtocol (class in mrjob.protocol) to_chunks() (in module mrjob.cat) to_lines() (in module mrjob.util) to_uri() (in module mrjob.parse)

touchz() (mrjob.fs.base.Filesystem method) translate_jobconf() (in module mrjob.compat) translate_jobconf_dict() (in module mrjob.compat) translate_jobconf_for_all_versions() (in module mrjob.compat) TZ

U

UltraJSONProtocol (class in mrjob.protocol) UltraJSONValueProtocol (class in mrjob.protocol) unarchive() (in module mrjob.util) unique() (in module mrjob.util)

UploadDirManager (class in mrjob.setup) uri() (mrjob.setup.UploadDirManager method) uses_yarn() (in module mrjob.compat)

V

version_gte() (in module mrjob.compat)

W

which() (in module mrjob.util)

WorkingDirManager (class in mrjob.setup)

Z

zip_dir() (in module mrjob.util)