---
title: "R Notebook"
output: html_notebook
---

# Packages for imputation

```{r}
library(VIM)
library(tidyverse)
```

# Imputation

Read the data

```{r}
bkl <- readRDS("../data/bkl-before-imp.rds") %>%
  mutate(woj = ifelse(woj == "00", NA, woj)) %>%
  mutate_at(vars(jezyk_angielski:komp_biurowe), as.logical) %>%
  mutate_at(vars(zrodlo,podregion,branza,forma_ogl), as.character)

bkl %>%
  filter(rok !=2012) %>%
  skimr::skim()
```

```{r}
dput(names(bkl))
```


Imputation of missing data



```{r}
set.seed(123)
imputed <- kNN(data = bkl,
               variable = c("zawod1", "woj", "nace"), 
               dist_var =  c("zawod1", "zrodlo", "forma_ogl", "branza", "jezyk_angielski", "jezyk_niemiecki", 
                             "komp_techniczne", "komp_matematyczne", "komp_kulturalne", "komp_komputerowe", 
                             "komp_kognitywne", "komp_kierownicze", "komp_interpersonalne", 
                             "komp_indywidualne", "komp_fizyczne", "komp_dyspozycyjne", "komp_biurowe"),
               k = 1, 
               addRF = TRUE,
               weights = "auto", 
               trace = TRUE)


```


```
Detected as categorical variable:

zrodlo,forma_ogl,zawod1,zawod6,woj,podregion,nace,sekcja_pkd2,branza,wyksztalcenie
Detected as ordinal variable:


Detected as numerical variable:

id,rok,zrodlo_dokladne
382items ofvariable:zawod1 imputed

286items ofvariable:woj imputed

29887items ofvariable:nace imputed

Time difference of 1.577767 hours
```

```{r}
saveRDS(imputed, "../data/bkl-after-imp.rds")
```

1. distribution of missing data
2. impute based on the full dataset
3. restrict only to Interent


After imputation 

```{r}
dim(imputed)
```


```{r}
imputed %>%
  filter(zrodlo == 1) %>%
  mutate(zawod2 = substr(zawod6,1,2),
         zawod2 = ifelse(is.na(zawod6), zawod1, substr(zawod6,1,2)),
         zawod2 = str_pad(string = zawod2, width = 2, side = "right", pad = "0"),
         zawod6 = ifelse(is.na(zawod6), zawod1, zawod6),
         zawod6 = str_pad(string = zawod6, width = 6, side = "right", pad = "0")) %>%
  select(id, rok, zawod1, zawod2, zawod6, woj, nace, komp_techniczne:komp_biurowe) %>%
  mutate_at(vars(komp_techniczne:komp_biurowe), as.integer) -> final_data
```

```{r}
saveRDS(final_data, "../data/bkl-final.rds")
```

```{r}
final_data  %>% 
  filter(rok != 2012) %>%
  count(zawod2) %>%
  filter(zawod2 %in% seq(10,90,10)) %>%
  summarise(sum(n))
```


```{r}
imputed %>%
  ungroup() %>%
  select(zrodlo, komp_techniczne:komp_biurowe) %>%
  gather(comp, vals, -zrodlo) %>%
  count(zrodlo, comp, vals) %>%
  add_count(zrodlo, comp, wt = n, name = "total") %>%
  mutate(p = n/total*100) %>%
  filter(vals) %>%
  select(zrodlo, comp, p) %>%
  mutate(comp = case_when(comp == "komp_kulturalne" ~ "Artistic",
                            comp == "komp_dyspozycyjne"~ "Availability",
                            comp == "komp_kognitywne" ~ "Cognitive",
                            comp == "komp_komputerowe" ~ "Computer",
                            comp == "komp_interpersonalne" ~ "Interpersonal",
                            comp == "komp_kierownicze" ~ "Managerial",
                            comp == "komp_matematyczne" ~ "Mathematical",
                            comp == "komp_biurowe"~ "Office",
                            comp == "komp_fizyczne" ~ "Physical",
                            comp == "komp_indywidualne"~ "Self-organization",
                            comp =="komp_techniczne"  ~ "Technical")) %>%
  spread(zrodlo, p) %>%
  left_join(imputed %>%
  ungroup() %>%
  select(zrodlo, komp_techniczne:komp_biurowe) %>%
  gather(comp, vals, -zrodlo) %>%
  count(comp, vals) %>%
  add_count(comp, wt = n, name = "total") %>%
  mutate(p = n/total*100) %>%
  filter(vals) %>%
  select(comp, p) %>%
  mutate(comp = case_when(comp == "komp_kulturalne" ~ "Artistic",
                            comp == "komp_dyspozycyjne"~ "Availability",
                            comp == "komp_kognitywne" ~ "Cognitive",
                            comp == "komp_komputerowe" ~ "Computer",
                            comp == "komp_interpersonalne" ~ "Interpersonal",
                            comp == "komp_kierownicze" ~ "Managerial",
                            comp == "komp_matematyczne" ~ "Mathematical",
                            comp == "komp_biurowe"~ "Office",
                            comp == "komp_fizyczne" ~ "Physical",
                            comp == "komp_indywidualne"~ "Self-organization",
                            comp =="komp_techniczne"  ~ "Technical"))) %>%
  xtable(digits = 1, caption = "Share of skills included in job offers by data source based on pooled data 2011, 2013 and 2014") %>%
  print.xtable(include.rownames = F, caption.placement = "top")
```

```{r}
imputed %>%
  ungroup() %>%
  select(zrodlo, komp_techniczne:komp_biurowe) %>%
  gather(comp, vals, -zrodlo) %>%
  count(zrodlo, comp, vals) %>%
  add_count(zrodlo, comp, wt = n, name = "total") %>%
  mutate(p = n/total*100) %>%
  filter(vals) %>%
  select(zrodlo, comp, p) %>%
  mutate(comp = case_when(comp == "komp_kulturalne" ~ "Artistic",
                            comp == "komp_dyspozycyjne"~ "Availability",
                            comp == "komp_kognitywne" ~ "Cognitive",
                            comp == "komp_komputerowe" ~ "Computer",
                            comp == "komp_interpersonalne" ~ "Interpersonal",
                            comp == "komp_kierownicze" ~ "Managerial",
                            comp == "komp_matematyczne" ~ "Mathematical",
                            comp == "komp_biurowe"~ "Office",
                            comp == "komp_fizyczne" ~ "Physical",
                            comp == "komp_indywidualne"~ "Self-organization",
                            comp =="komp_techniczne"  ~ "Technical")) %>%
  spread(zrodlo, p) %>%
  left_join(imputed %>%
  ungroup() %>%
  select(zrodlo, komp_techniczne:komp_biurowe) %>%
  gather(comp, vals, -zrodlo) %>%
  count(comp, vals) %>%
  add_count(comp, wt = n, name = "total") %>%
  mutate(p = n/total*100) %>%
  filter(vals) %>%
  select(comp, p)) %>%
  select(online=`1`, pup = `2`) %>%
  cor(m = "s")
```

