knitr::opts_chunk$set(
warning = FALSE, # show warnings during codebook generation
message = FALSE, # show messages during codebook generation
error = TRUE, # do not interrupt codebook generation in case of errors,
# usually better for debugging
echo = TRUE # show R code
)
ggplot2::theme_set(ggplot2::theme_bw())
# set base directory
basedir = "/home/ecco_rais/data/clean/RAIS-homogenized/output/"
# adjust as necessary
startyear = 2003
endyear = 2004
# libraries
library(codebook)
library(rio)
# Start the codebook loop
#for ( year in startyear:endyear) {
year = "2010"
# for CSV
codebook_data <- fread(file.path(basedir, paste0("harmo_", year, ".csv")))
# omit the following lines, if your missing values are already properly labelled
codebook_data <- detect_missing(codebook_data,
only_labelled = TRUE, # only labelled values are autodetected as
# missing
negative_values_are_missing = FALSE, # negative values are missing values
ninety_nine_problems = TRUE, # 99/999 are missing values, if they
# are more than 5 MAD from the median
)
# If you are not using formr, the codebook package needs to guess which items
# form a scale. The following line finds item aggregates with names like this:
# scale = scale_1 + scale_2R + scale_3R
# identifying these aggregates allows the codebook function to
# automatically compute reliabilities.
# However, it will not reverse items automatically.
codebook_data <- detect_scales(codebook_data)
## Warning in detect_scales(codebook_data): cnae items found, but no aggregate
## Warning in detect_scales(codebook_data): active items found, but no aggregate
## Warning in detect_scales(codebook_data): cbo items found, but no aggregate
codebook_data <- as.data.table(codebook_data)
codebook(codebook_data)
## Warning in max(f): no non-missing arguments to max; returning -Inf
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning in max(f): no non-missing arguments to max; returning -Inf
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
## Warning: Couldn't find skimmers for class: integer64; No user-defined `sfl` provided. Falling
## back to `character`.
Dataset name: codebook_data
The dataset has N=66747302 rows and 47 columns. 31474058 rows have no missing values on any column.
|
#Variables
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
adm_date | numeric | 0 | 1 | 1e+06 | 4082010 | 3.1e+07 | 8426056 | 8468636 | ▇▂▂▂▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
separation_cause | numeric | 0 | 1 | 0 | 0 | 80 | 5.229521 | 8.628535 | ▇▁▁▁▁ | NA |
## Error in `ggplot2::geom_histogram()`:
## ! Problem while computing position.
## ℹ Error occurred in the 1st layer.
## Caused by error in `if (...) NULL`:
## ! missing value where TRUE/FALSE needed
0 missing values.
name | data_type | n_missing | complete_rate | n_unique | empty | min | max | whitespace | label |
---|---|---|---|---|---|---|---|---|---|
cei | character | 0 | 1 | 47226 | 0 | 1 | 21 | 0 | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cnae20 | numeric | 0 | 1 | 1113 | 47890 | 99008 | 54591.72 | 26405.17 | ▃▂▇▂▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cnae95 | numeric | 0 | 1 | 1112 | 52710 | 99007 | 54952.4 | 23072.28 | ▃▂▇▇▂ | NA |
## Error in `ggplot2::geom_histogram()`:
## ! Problem while computing position.
## ℹ Error occurred in the 1st layer.
## Caused by error in `if (...) NULL`:
## ! missing value where TRUE/FALSE needed
0 missing values.
name | data_type | n_missing | complete_rate | n_unique | empty | min | max | whitespace | label |
---|---|---|---|---|---|---|---|---|---|
cpf | character | 0 | 1 | 53438884 | 0 | 1 | 21 | 0 | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
dob | numeric | 0 | 1 | 1e+06 | 1.5e+07 | 3.1e+07 | 15644527 | 8782268 | ▇▇▇▇▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
active1231 | numeric | 0 | 1 | 0 | 1 | 1 | 0.6602268 | 0.4736321 | ▅▁▁▁▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
sex | numeric | 0 | 1 | 0 | 0 | 1 | 0.3983789 | 0.4895643 | ▇▁▁▁▅ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
schooling | numeric | 0 | 1 | 1 | 7 | 11 | 6.281065 | 1.860445 | ▂▃▇▃▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
hired_hours | numeric | 0 | 1 | 1 | 44 | 44 | 41.36893 | 6.204104 | ▁▁▁▁▇ | NA |
## Error in `ggplot2::geom_histogram()`:
## ! Problem while computing stat.
## ℹ Error occurred in the 1st layer.
## Caused by error in `seq_len()`:
## ! argument must be coercible to non-negative integer
0 missing values.
name | data_type | n_missing | complete_rate | n_unique | empty | min | max | whitespace | label |
---|---|---|---|---|---|---|---|---|---|
firmID | character | 0 | 1 | 3359136 | 0 | 15 | 21 | 0 | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
alvara_ind | numeric | 0 | 1 | 1 | 2 | 2 | 1.999798 | 0.014196 | ▁▁▁▁▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cei_avail | numeric | 0 | 1 | 0 | 0 | 1 | 0.0231929 | 0.1505158 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
pat | numeric | 0 | 1 | 0 | 0 | 1 | 0.2874332 | 0.4525653 | ▇▁▁▁▃ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
simples | numeric | 0 | 1 | 0 | 0 | 1 | 0.2203253 | 0.414466 | ▇▁▁▁▂ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
termination_month | numeric | 0 | 1 | 0 | 0 | 12 | 2.314386 | 3.815496 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
muni | numeric | 0 | 1 | 110001 | 351870 | 530010 | 345254.8 | 85297.81 | ▁▂▇▂▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
nationality | numeric | 0 | 1 | 10 | 10 | 50 | 10.03229 | 0.9823145 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
legal_form | numeric | 0 | 1 | 1015 | 2062 | 5037 | 2044.616 | 659.8958 | ▂▇▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | n_unique | empty | min | max | whitespace | label |
---|---|---|---|---|---|---|---|---|---|
name | character | 0 | 1 | 38634999 | 0 | 5 | 52 | 0 | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
ctps | numeric | 0 | 1 | 0 | 58120 | 1e+08 | 815545.1 | 3188093 | ▇▁▁▁▁ | NA |
629203 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cbo02 | numeric | 629203 | 0.9905734 | 111215 | 514320 | 992225 | 524140.4 | 199658.4 | ▃▇▇▇▂ | NA |
647293 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cbo94 | numeric | 647293 | 0.9903023 | 1105 | 53110 | 99990 | 53813.65 | 28625.1 | ▅▇▇▃▇ | NA |
## Error in `ggplot2::geom_histogram()`:
## ! Problem while computing stat.
## ℹ Error occurred in the 1st layer.
## Caused by error in `seq_len()`:
## ! argument must be coercible to non-negative integer
0 missing values.
name | data_type | n_missing | complete_rate | n_unique | empty | min | max | whitespace | label |
---|---|---|---|---|---|---|---|---|---|
pis | character | 0 | 1 | 53771613 | 0 | 13 | 21 | 0 | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
disabled | numeric | 0 | 1 | 0 | 1 | 1 | 0.9939972 | 0.077245 | ▁▁▁▁▇ | NA |
9588080 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
race_color | numeric | 9588080 | 0.8563525 | 1 | 2 | 9 | 4.300151 | 2.905527 | ▇▁▁▁▅ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cnpj_root | numeric | 0 | 1 | 0 | 9380370 | 1e+08 | 24262570 | 27537522 | ▇▁▂▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
earn_dec | numeric | 0 | 1 | 0 | 680 | 76490 | 1154.704 | 2268.721 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
earn_dec_mw | numeric | 0 | 1 | 0 | 1.3 | 150 | 2.260952 | 4.447607 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
mean_earn | numeric | 0 | 1 | 0 | 839 | 76501 | 1431.966 | 2165.806 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
mean_earn_mw | numeric | 0 | 1 | 0 | 1.6 | 150 | 2.799812 | 4.246286 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
hired_wage | numeric | 0 | 1 | 0 | 684 | 1e+07 | 1070.02 | 4949.645 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
cnae20sub | numeric | 0 | 1 | 111301 | 4789099 | 9900800 | 5459181 | 2640515 | ▃▂▇▂▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
establishment_size | numeric | 0 | 1 | 0 | 6 | 9 | 5.393574 | 2.799732 | ▃▅▆▅▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
tenure | numeric | 0 | 1 | 0 | 15 | 600 | 44.97137 | 72.96792 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
adm_type | numeric | 0 | 1 | 0 | 0 | 10 | 0.7599076 | 1.032914 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
firmID_type | numeric | 0 | 1 | 0 | 1 | 1 | 0.9671163 | 0.1783322 | ▁▁▁▁▇ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
wage_type | numeric | 0 | 1 | 1 | 1 | 7 | 1.420927 | 1.257718 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
disability_type | numeric | 0 | 1 | 0 | 0 | 6 | 0.0124545 | 0.2035444 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
contract_type | numeric | 0 | 1 | 10 | 10 | 97 | 16.38293 | 14.16239 | ▇▂▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
last_wage | numeric | 0 | 1 | 0 | 1.7 | 150 | 3.090763 | 5.395119 | ▇▁▁▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
yr | numeric | 0 | 1 | 2010 | 2010 | 2010 | 2010 | 0 | ▁▁▇▁▁ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
termination_year | numeric | 0 | 1 | 2010 | 2010 | 2010 | 2010 | 0 | ▁▁▇▁▁ | NA |
28979625 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
adm_day | numeric | 28979625 | 0.5658308 | 1 | 13 | 31 | 13.69469 | 7.799867 | ▇▇▇▅▂ | NA |
4840436 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
adm_month | numeric | 4840436 | 0.9274812 | 1 | 6 | 12 | 6.054083 | 3.214172 | ▇▅▅▆▅ | NA |
0 missing values.
name | data_type | n_missing | complete_rate | min | median | max | mean | sd | hist | label |
---|---|---|---|---|---|---|---|---|---|---|
adm_year | numeric | 0 | 1 | 1929 | 2009 | 2010 | 2006.609 | 6.010201 | ▁▁▁▁▇ | NA |
The following JSON-LD can be found by search engines, if you share this codebook publicly on the web.
{
"name": "codebook_data",
"datePublished": "2024-06-01",
"description": "The dataset has N=66747302 rows and 47 columns.\n31474058 rows have no missing values on any column.\n\n\n## Table of variables\nThis table contains variable names, labels, and number of missing values.\nSee the complete codebook for more.\n\n|name |label | n_missing|\n|:------------------|:-----|---------:|\n|adm_date |NA | 0|\n|separation_cause |NA | 0|\n|cei |NA | 0|\n|cnae20 |NA | 0|\n|cnae95 |NA | 0|\n|cpf |NA | 0|\n|dob |NA | 0|\n|active1231 |NA | 0|\n|sex |NA | 0|\n|schooling |NA | 0|\n|hired_hours |NA | 0|\n|firmID |NA | 0|\n|alvara_ind |NA | 0|\n|cei_avail |NA | 0|\n|pat |NA | 0|\n|simples |NA | 0|\n|termination_month |NA | 0|\n|muni |NA | 0|\n|nationality |NA | 0|\n|legal_form |NA | 0|\n|name |NA | 0|\n|ctps |NA | 0|\n|cbo02 |NA | 629203|\n|cbo94 |NA | 647293|\n|pis |NA | 0|\n|disabled |NA | 0|\n|race_color |NA | 9588080|\n|cnpj_root |NA | 0|\n|earn_dec |NA | 0|\n|earn_dec_mw |NA | 0|\n|mean_earn |NA | 0|\n|mean_earn_mw |NA | 0|\n|hired_wage |NA | 0|\n|cnae20sub |NA | 0|\n|establishment_size |NA | 0|\n|tenure |NA | 0|\n|adm_type |NA | 0|\n|firmID_type |NA | 0|\n|wage_type |NA | 0|\n|disability_type |NA | 0|\n|contract_type |NA | 0|\n|last_wage |NA | 0|\n|yr |NA | 0|\n|termination_year |NA | 0|\n|adm_day |NA | 28979625|\n|adm_month |NA | 4840436|\n|adm_year |NA | 0|\n\n### Note\nThis dataset was automatically described using the [codebook R package](https://rubenarslan.github.io/codebook/) (version 0.9.2).",
"keywords": ["adm_date", "separation_cause", "cei", "cnae20", "cnae95", "cpf", "dob", "active1231", "sex", "schooling", "hired_hours", "firmID", "alvara_ind", "cei_avail", "pat", "simples", "termination_month", "muni", "nationality", "legal_form", "name", "ctps", "cbo02", "cbo94", "pis", "disabled", "race_color", "cnpj_root", "earn_dec", "earn_dec_mw", "mean_earn", "mean_earn_mw", "hired_wage", "cnae20sub", "establishment_size", "tenure", "adm_type", "firmID_type", "wage_type", "disability_type", "contract_type", "last_wage", "yr", "termination_year", "adm_day", "adm_month", "adm_year"],
"@context": "http://schema.org/",
"@type": "Dataset",
"variableMeasured": [
{
"name": "adm_date",
"@type": "propertyValue"
},
{
"name": "separation_cause",
"@type": "propertyValue"
},
{
"name": "cei",
"@type": "propertyValue"
},
{
"name": "cnae20",
"@type": "propertyValue"
},
{
"name": "cnae95",
"@type": "propertyValue"
},
{
"name": "cpf",
"@type": "propertyValue"
},
{
"name": "dob",
"@type": "propertyValue"
},
{
"name": "active1231",
"@type": "propertyValue"
},
{
"name": "sex",
"@type": "propertyValue"
},
{
"name": "schooling",
"@type": "propertyValue"
},
{
"name": "hired_hours",
"@type": "propertyValue"
},
{
"name": "firmID",
"@type": "propertyValue"
},
{
"name": "alvara_ind",
"@type": "propertyValue"
},
{
"name": "cei_avail",
"@type": "propertyValue"
},
{
"name": "pat",
"@type": "propertyValue"
},
{
"name": "simples",
"@type": "propertyValue"
},
{
"name": "termination_month",
"@type": "propertyValue"
},
{
"name": "muni",
"@type": "propertyValue"
},
{
"name": "nationality",
"@type": "propertyValue"
},
{
"name": "legal_form",
"@type": "propertyValue"
},
{
"name": "name",
"@type": "propertyValue"
},
{
"name": "ctps",
"@type": "propertyValue"
},
{
"name": "cbo02",
"@type": "propertyValue"
},
{
"name": "cbo94",
"@type": "propertyValue"
},
{
"name": "pis",
"@type": "propertyValue"
},
{
"name": "disabled",
"@type": "propertyValue"
},
{
"name": "race_color",
"@type": "propertyValue"
},
{
"name": "cnpj_root",
"@type": "propertyValue"
},
{
"name": "earn_dec",
"@type": "propertyValue"
},
{
"name": "earn_dec_mw",
"@type": "propertyValue"
},
{
"name": "mean_earn",
"@type": "propertyValue"
},
{
"name": "mean_earn_mw",
"@type": "propertyValue"
},
{
"name": "hired_wage",
"@type": "propertyValue"
},
{
"name": "cnae20sub",
"@type": "propertyValue"
},
{
"name": "establishment_size",
"@type": "propertyValue"
},
{
"name": "tenure",
"@type": "propertyValue"
},
{
"name": "adm_type",
"@type": "propertyValue"
},
{
"name": "firmID_type",
"@type": "propertyValue"
},
{
"name": "wage_type",
"@type": "propertyValue"
},
{
"name": "disability_type",
"@type": "propertyValue"
},
{
"name": "contract_type",
"@type": "propertyValue"
},
{
"name": "last_wage",
"@type": "propertyValue"
},
{
"name": "yr",
"@type": "propertyValue"
},
{
"name": "termination_year",
"@type": "propertyValue"
},
{
"name": "adm_day",
"@type": "propertyValue"
},
{
"name": "adm_month",
"@type": "propertyValue"
},
{
"name": "adm_year",
"@type": "propertyValue"
}
]
}`
# } # end year loop