changed connection name

Main.R

 #####
 #
-# Load simulated Watershed data of SIMO and create CSV.files for further analysis
+# Convert SIMO simulated data to CSV files
 # 2020-02-21
 #
 #####
 
+rm(list = ls())  # remove all files in memory
 
 
 library(dplyr)
 
-# this is my big change!!!
 
 ### Set the working path to this R-project
-path <- paste0(getwd(),"/")
+path <- paste0(getwd())
+
+
+### Define the input & output folders
+# ------------------------------------
+
+#   Input: contains all .db and will convert them to 
+#   to .csv in 'output' folder
+inputFolder  <- paste(path, "input_data", sep = "/")
+outputFolder <- paste(path, "output", sep = "/")
 
 
 ### load some parameter
 # Management regimes and their abbreviation, they are merged to the data by the branching group (script loadDB.R)
-regime <- read.csv(paste0(path, "params/regimes.csv"), sep = ",", stringsAsFactors = FALSE)
+regime <- read.csv(paste0(path, "/params/regimes.csv"), 
+                   sep = ",", 
+                   stringsAsFactors = FALSE)
 
 
 ### Simulation variant that will be load
 ### The corresponding db files need to be stored in folder "input_data"
+### The output .csv files will be stored in folder "output" 
 ### specify which one should be read:
 
 
@@ -52,21 +64,45 @@ regime <- read.csv(paste0(path, "params/regimes.csv"), sep = ",", stringsAsFacto
 #                   "CC0_p_canesm"       NEW: RCP scenario 0
 
 
-sim_variant <- "CC0_p_canesm"  
+# Select the variation 
+#sim_variant <- "CC0_p_canesm"
+sim_variant <- "without"
+
+# Vector to keep all variants for future use:
+all.variants <- c("CC45", 
+                  "CC85", 
+                  "without", 
+                  "CC45_p", 
+                  "CC85_p", 
+                  "CC26_p", 
+                  "CC0_p", 
+                  "CC45_p_canesm", 
+                  "CC85_p_canesm", 
+                  "CC26_p_canesm", 
+                  "CC0_p_canesm")
 
 
 ### Define the names of the databases (SIMO-output for 10 watersheds) that will be imported
 # It is used in the skripts "structure_SIMO_rslDB_FBE.R" and "loadDB.R"
-db_names <- c("MV_Hartola", 
-              "MV_Kitee", 
-              "MV_Korsnas",
-              "MV_Parikkala",
-              "MV_Pori",
-              "MV_Pyhtaa",
-              "MV_Raasepori",
-              "MV_Simo",
-              "MV_Vaala",
-              "MV_Voyri")
+#db_names <- c(#"MV_Hartola", 
+              #"MV_Kitee", 
+ #             "MV_Korsnas" #,
+              #"MV_Parikkala",
+              #"MV_Pori",
+              #"MV_Pyhtaa",
+              #"MV_Raasepori",
+              #"MV_Simo",
+              #"MV_Vaala",
+              #"MV_Voyri"
+          #    )
+
+# List all .db in 'input'
+db_input <- list.files(inputFolder,
+                       pattern = ".db$")
+
+
+#Remove the ending ".db" from the list to create a database name
+db_names <- gsub(".db", "", db_input)
 
 
 ### Restructure the SQL database. 
@@ -83,16 +119,16 @@ first_load = FALSE
 if(first_load == TRUE){
 
   # If one of the follwing simulation variants is read ...
-  if(sim_variant %in% c("CC45", "CC85", "without", "CC45_p", "CC85_p", "CC26_p", "CC0_p", "CC45_p_canesm", "CC85_p_canesm", "CC26_p_canesm", "CC0_p_canesm")) {
+  if(sim_variant %in% all.variants ) {
     
     # Run the script with the SQL query for all management regimes
-    source(paste0(path, "structure_SIMO_rslDB_FBE.R"))  
+    source(paste0(path, "/structure_SIMO_rslDB_FBE.R"))  
   
     } else {
     
       # Otherwise, run the script witht the SQL query for Set aside without deadwood extraction (...SA)
       # This needs a different SQL query, since the database does not contain harvest information
-      source(paste0(path, "structure_SIMO_rslDB_SA.R")) # Query 
+      source(paste0(path, "/structure_SIMO_rslDB_SA.R")) # Query 
       
       }
 
@@ -125,27 +161,32 @@ columns <-  paste0("id,
                    CARBON_STORAGE")
 
 
+# Create the final CSV
+# -------------------------
+
+source(paste(path, "loadDB.R", sep = "/")) 
+
 ### !!! CSV files are stored unter output
 #
 # If data/columns have already been loaded and saved as csv file  under ../output:                csv_exist = TRUE
 # If data is loaded for the firest time OR "new columns have to be loaded" (takes some time):     csv_exits = FALSE 
 
 
-csv_exist = TRUE
-
-
+#csv_exist = TRUE
+#
+#
 ### the following lines do not need any changes ###
-if(csv_exist == FALSE){
-  
+#if(csv_exist == FALSE){
+ # 
   # If FALSE run the following script 
-  source(paste0(path, "loadDB.R")) 
- 
-  } else {
-  
-    # If TRUE read the CSV file that contains all watersheds (GPKGs)
-    rslt <- read.csv(paste0(path, "output/rslt_", sim_variant, "_all.csv" ), sep = ";", header = TRUE, stringsAsFactors = FALSE)
-
-    }
+  #source(paste0(path, "loadDB.R")) 
+ #
+  #} else {
+  #
+   # # If TRUE read the CSV file that contains all watersheds (GPKGs)
+    #rslt <- read.csv(paste0(path, "output/rslt_", sim_variant, "_all.csv" ), sep = ";", header = TRUE, stringsAsFactors = FALSE)
+#
+ #   }
 ###
 
 

loadDB.R

@@ -16,84 +16,105 @@ library(dplyr)
 
 
 
-# # Variant 1: create a single dataframe for each SQL Database (selected columns of table UNIT)
-# # Dataframes are given the names of the DB and column "test" is added with the name of the DB
-# 
-# for (name in db_names){
-#   
-#   # name = "MV_Hartola"
-#   db <- dbConnect(dbDriver("SQLite"), dbname = paste0(path,"input_data/simulated_", sim_variant, "_", name, "_rsu.db"))
-#   rsl  <- dbGetQuery( db, paste0("select ", columns, " from UNIT"))
-#   rsl$gpkg <- name
-#   dbDisconnect(db)
-#   
-#   ### Add the abbreviation of the regimes
-#   rsl <- rsl %>% 
-#     left_join(regime, by = "branching_group", all.x = TRUE)
-#   
-#   ### Filter those stands that caused errors during the simulation with SIMO
-#   # import csv file that contains the error stands
-#   error_stands <- read.csv(paste0(path, "params/errors_watersheds.csv"), sep = ",", header = TRUE, stringsAsFactors = FALSE)
-#   error_stands$id <- as.character(error_stands$id)
-# 
-#   rsl <- rsl %>%
-#     anti_join(error_stands, by = c("id", "gpkg"))
-#   
-#   
-#   ### Rename set aside scenario if it considers deadwood extraction
-#   if(sim_variant %in% c("CC45", "CC85", "without", "CC45_p", "CC85_p", "CC26_p", "CC0_p", "CC45_p_canesm", "CC85_p_canesm", "CC26_p_canesm", "CC0_p_canesm")) {
-#     
-#    rsl <- rsl %>% mutate(regime = ifelse(regime %in% "SA", "SA_DWextract", regime))
-#     
-#   }
-#   
-#   
-#   write.table(rsl, paste0(path, "output/rsl_",sim_variant,"_",name,".csv" ), sep = ";", row.names = F, col.names = TRUE)
-#   
-#   assign( paste("rsl", name, sep="_"), rsl)
-#   rm(db, rsl)
-#   
-# }
-
-
-# Variant 2: create one hughe dataframe combining all SQL databases (selected columns of table UNIT)
-# Different DBs are indicated by an additional column "test"
-
-rslt <- NULL
+# Variant 1: create a single dataframe for each SQL Database (selected columns of table UNIT)
+# Dataframes are given the names of the DB and column "test" is added with the name of the DB
 
 for (name in db_names){
-  db <- dbConnect(dbDriver("SQLite"), dbname = paste0(path,"input_data/simulated_", sim_variant, "_" , name, "_rsu.db"))
+
+  #name = "simulated_without_MV_Korsnas_Wind_NO_1-15"
+  #db <- dbConnect(dbDriver("SQLite"), dbname = paste0(path,"input_data/simulated_", sim_variant, "_", name, "_rsu.db"))
+  db <- dbConnect(dbDriver("SQLite"),
+                  dbname = paste(inputFolder, 
+                                  paste0(name, ".db"), sep = "/"))
+  #db <- dbConnect(dbDriver("SQLite"),
+   #               dbname = paste0(path, "input_data/simulated_without_MV_Korsnas_Wind_NO_1-15.db"))
+  
   rsl  <- dbGetQuery( db, paste0("select ", columns, " from UNIT"))
   rsl$gpkg <- name
   dbDisconnect(db)
-  rslt <- rbind(rslt, rsl)
-  rm(db, rsl) 
-}  
 
-### Add the abbreviation of the regimes
-rslt <- rslt %>% 
-  left_join(regime, by= "branching_group", all.X = TRUE)
+  ### Add the abbreviation of the regimes
+  rsl <- rsl %>%
+    left_join(regime, by = "branching_group", all.x = TRUE)
 
+  ### Filter those stands that caused errors during the simulation with SIMO
+  # import csv file that contains the error stands
+  error_stands <- read.csv(paste0(path, "/params/errors_watersheds.csv"), sep = ",", header = TRUE, stringsAsFactors = FALSE)
+  error_stands$id <- as.character(error_stands$id)
+
+  rsl <- rsl %>%
+    anti_join(error_stands, by = c("id", "gpkg"))
+
+
+  ### Rename set aside scenario if it considers deadwood extraction
+  if(sim_variant %in% all.variants) {
+
+   rsl <- rsl %>% mutate(regime = ifelse(regime %in% "SA", "SA_DWextract", regime))
+
+  }
+
+  # Export the final table
+  print(paste0("writing csv for ", name))
+  write.table(rsl, 
+              paste0(outputFolder, "/", name, ".csv" ), 
+              sep = ";", 
+              row.names = F, 
+              col.names = TRUE)
+
+  #assign( paste("rsl", name, sep="_"), rsl)
+  #rm(db, rsl)
+  #print("csv exported")
 
-### Rename set aside scenario if it considers deadwood extraction
-if(sim_variant %in% c("CC45", "CC85", "without", "CC45_p", "CC85_p", "CC26_p", "CC0_p", "CC45_p_canesm", "CC85_p_canesm", "CC26_p_canesm", "CC0_p_canesm")) {
-  
-  rslt <- rslt %>% mutate(regime = ifelse(regime %in% "SA", "SA_DWextract", regime))
-  
 }
 
 
+# Variant 2: create one huge dataframe combining all SQL databases (selected columns of table UNIT)
+# Different DBs are indicated by an additional column "test"
+
+# rslt <- NULL
+# 
+# for (name in db_names){
+#   #name = "MV_Korsnas"
+#   db <- dbConnect(dbDriver("SQLite"),
+#                   dbname = paste0(inputFolder, name, ".db"))
+#                   #dbname = paste0(path, "input_data/simulated_without_MV_Korsnas_Wind_NO_1-15.db"))
+#                   #dbname = paste0(path,"input_data/simulated_", sim_variant, "_" , name, "_rsu.db"))
+#   rsl  <- dbGetQuery( db, paste0("select ", columns, " from UNIT"))
+#   rsl$gpkg <- name
+#   dbDisconnect(db)
+#   rslt <- rbind(rslt, rsl)
+#   rm(db, rsl) 
+# }  
+# 
+# ### Add the abbreviation of the regimes
+# rslt <- rslt %>% 
+#   left_join(regime, by= "branching_group", all.X = TRUE)
+# 
+# 
+# ### Rename set aside scenario if it considers deadwood extraction
+# if(sim_variant %in% all.variants) {
+#   
+#   rslt <- rslt %>% mutate(regime = ifelse(regime %in% "SA", "SA_DWextract", regime))
+#   
+# }
+# 
+# 
 ### Filter those stands that caused errors during the simulation with SIMO
 # import csv file that contains the error stands
-error_stands <- read.csv(paste0(path, "params/errors_watersheds.csv"), sep = ",", header = TRUE, stringsAsFactors = FALSE)
-error_stands$id <- as.character(error_stands$id)
-
-rslt <- rslt %>%
-  anti_join(error_stands, by = c("id", "gpkg"))
-
-### write table
-write.table(rslt, paste0(path, "output/rslt_", sim_variant, "_all.csv" ), sep = ";", row.names = F, col.names = TRUE)
+# error_stands <- read.csv(paste0(path, "params/errors_watersheds.csv"), sep = ",", header = TRUE, stringsAsFactors = FALSE)
+# error_stands$id <- as.character(error_stands$id)
+# 
+# rslt <- rslt %>%
+#   anti_join(error_stands, by = c("id", "gpkg"))
+# 
+# ### write table
+# write.table(rslt, paste0(path, "output/rslt_", sim_variant, "_all.csv" ), sep = ";", row.names = F, col.names = TRUE)
 
+# write.table(rsl, 
+#             paste0(path, "/", name, ".csv" ), 
+#             sep = ";", 
+#             row.names = F, 
+#             col.names = TRUE)
 
 
 

structure_SIMO_rslDB_FBE.R

@@ -125,18 +125,40 @@ create_table_UNIT <- 'Create Table UNIT AS SELECT u.*,
 
 
 ##### For each "db_names", defined in main.R ...
-
+for (name in db_names) {
+  print(name)
+  db <- dbConnect(dbDriver("SQLite"),
+                  dbname = paste(inputFolder, 
+                                 paste0(name, ".db"), sep = "/"))
+  print(db)
+  dbDisconnect(db)
+}
 
 
 for (name in db_names){
-  name = "MV_Korsnas"
+  #print(name)
+  
+  #name = "simulated_without_MV_Korsnas_Wind_NO_1-15"
   # Connect to the database
   #con <- dbConnect(dbDriver("SQLite"), 
    #                dbname = paste0(path,"input_data/simulated_", sim_variant,"_" ,name , "_rsu.db"))
   
+  #db <- dbConnect(dbDriver("SQLite"),
+   #               dbname = paste0(path, "input_data/simulated_without_MV_Korsnas_Wind_NO_1-15.db"))
+ 
+  #db <- dbConnect(dbDriver("SQLite"),
+   #               dbname = paste(inputFolder, paste0(name, ".db"), sep = "/"))
+  
+  
+  
+  
+  
+  
   db <- dbConnect(dbDriver("SQLite"),
-                  dbname = paste0(path, "input_data/simulated_without_MV_Korsnas_Wind_NO_1-15.db"))
+                  dbname = paste(inputFolder, 
+                                 paste0(name, ".db"), sep = "/"))
   
+   
   # If the following tables already exist, for which the query is defined, remove them
   tab_to_delete <- c("OPERS2", "OPERS3", "max_v", "UNIT")
   
@@ -152,7 +174,7 @@ for (name in db_names){
                     create_table_UNIT)
   
   for(i in query_to_run){
-    #i = create_table_UNIT
+    
     dbExecute(db, i)
   }
   

structure_SIMO_rslDB_SA.R

@@ -71,7 +71,13 @@ for (name in db_names){
   # name = "MV_Hartola"
   
   # Connect to the database
-  con <- dbConnect(dbDriver("SQLite"), dbname = paste0(path,"input_data/simulated_", sim_variant,"_" ,name , "_rsu.db"))
+  #con <- dbConnect(dbDriver("SQLite"), dbname = paste0(path,"input_data/simulated_", sim_variant,"_" ,name , "_rsu.db"))
+ # db <- dbConnect(dbDriver("SQLite"),
+  #                dbname = paste(inputFolder, paste0(name, ".db"), sep = "/"))
+  db <- dbConnect(dbDriver("SQLite"),
+                  dbname = paste(inputFolder, 
+                                 paste0(name, ".db"), sep = "/"))
+  
   
   # If the following tables already exist, for which the query is defined, remove them
   tab_to_delete <- c("OPERS2", "OPERS3", "max_v", "UNIT")