text: # -*- coding: utf-8 -*-
  """Sales_Departmentsss.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/162Ti1FAmllqvMJa1aX3u02IY1JPnM6OW
  
  ### IMPORTING LIBRARIES AND DATASET
  """
  
  # Commented out IPython magic to ensure Python compatibility.
  import pandas as pd
  import numpy as np
  import seaborn as sns
  import matplotlib.pyplot as plt
  import datetime
  
  import warnings
  warnings.filterwarnings('ignore')
  
  # %matplotlib inline
  
  from google.colab import files
  
  
  uploaded = files.upload()
  
  #loading dataset
  sales_train_df = pd.read_csv('train.csv')
  
  sales_train_df.head(5)
  # almost a million observation 
  # 1115 unique stores 
  # Note that sales is the target variable (that's what we are trying to predict) 
  
  # Id: transaction ID (combination of Store and date) 
  # Store: unique store Id
  # Sales: sales/day, this is the target variable 
  # Customers: number of customers on a given day
  # Open: Boolean to say whether a store is open or closed (0 = closed, 1 = open)
  # Promo: describes if store is running a promo on that day or not
  # StateHoliday: indicate which state holiday (a = public holiday, b = Easter holiday, c = Christmas, 0 = None)
  # SchoolHoliday: indicates if the (Store, Date) was affected by the closure of public schools
  
  #- Data Source: https://www.kaggle.com/c/rossmann-store-sales/data
  
  sales_train_df.tail(10)
  
  sales_train_df.info()
  # 9 columns in total 
  # 8 features, each contains 1017209 data points
  # 1 target variable (sales)
  
  sales_train_df.describe()
  # Average sales amount per day = 5773 Euros, minimum sales per day = 0, maximum sales per day = 41551 
  # Average number of customers = 633, minimum number of customers = 0, maximum number of customers = 7388
  
  """MPORTING STORE INFORMATION DATA"""
  
  store_info_df = pd.read_csv('store.csv')
  
  # StoreType: categorical variable to indicate type of store (a, b, c, d)
  # Assortment: describes an assortment level: a = basic, b = extra, c = extended
  # CompetitionDistance (meters): distance to closest competitor store
  # CompetitionOpenSince [Month/Year]: provides an estimate of the date when competition was open
  # Promo2: Promo2 is a continuing and consecutive promotion for some stores (0 = store is not participating, 1 = store is participating)
  # Promo2Since [Year/Week]: date when the store started participating in Promo2
  # PromoInterval: describes the consecutive intervals Promo2 is started, naming the months the promotion is started anew. E.g. "Feb,May,Aug,Nov" means each round starts in February, May, August, November of any given year for that store
  
  store_info_df.head(5)
  
  # Let's do the same for the store_info_df data
  # Note that the previous dataframe includes the transactions recorded per day (in millions)
  # This dataframe only includes information about the unique 1115 stores that are part of this study 
  store_info_df.info()
  
  store_info_df.describe()
  # on average, the competition distance is 5404 meters away (5.4 kms)
  
  """### EXPLORING DATASET
  
  EXPLORING SALES TRAINING DATA
  """
  
  # Let's see if we have any missing data, luckily we don't!
  sns.heatmap(sales_train_df.isnull(), yticklabels = False, cbar = False, cmap="Blues")
  
  sales_train_df.hist(bins = 30, figsize = (20,20), color = 'r');
  plt.savefig('histplot_of_features.png')
  # Average 600 customers per day, maximum is 4500 (note that we can't see the outlier at 7388!)
  # Data is equally distibuted across various Days of the week (~150000 observations x 7 day = ~1.1 million observation) 
  # Stores are open ~80% of the time
  # Data is equally distributed among all stores (no bias)
  # Promo #1 was running ~40% of the time 
  # Average sales around 5000-6000 Euros
  # School holidays are around ~18% of the time
  
  sales_train_df['Customers'].max()
  
  # Let's see how many stores are open and closed! 
  closed_train_df        = sales_train_df[sales_train_df['Open'] == 0]
  open_train_df          = sales_train_df[sales_train_df['Open'] == 1]
  
  # Counting the number of stores that are open and closed
  print("Total =", len(sales_train_df))
  print("Number of closed stores =", len(closed_train_df))
  print("Number of open stores =", len(open_train_df))
  
  # only keep open stores and remove closed stores
  sales_train_df = sales_train_df[sales_train_df['Open'] == 1]
  
  sales_train_df
  
  # Let's drop the open column since it has no meaning now
  sales_train_df.drop(['Open'], axis=1, inplace=True)
  
  sales_train_df
  
  sales_train_df.describe()
  # Average sales = 6955 Euros,	average number of customers = 762	(went up)
  
  """EXPLORING STORES INFORMATION DATA"""
  
  # Let's see if we have any missing data in the store information dataframe!
  sns.heatmap(store_info_df.isnull(), yticklabels = False, cbar = False, cmap="Blues")
  plt.savefig('null_values.png')
  
  # Let's take a look at the missing values in the 'CompetitionDistance'
  # Only 3 rows are missing 
  store_info_df[store_info_df['CompetitionDistance'].isnull()]
  
  # Let's take a look at the missing values in the 'CompetitionOpenSinceMonth'
  # many rows are missing = 354 (almost one third of the 1115 stores)
  store_info_df[store_info_df['CompetitionOpenSinceMonth'].isnull()]
  
  store_info_df[ store_info_df['Promo2'] == 0]
  
  # It seems like if 'promo2' is zero, 'promo2SinceWeek', 'Promo2SinceYear', and 'PromoInterval' information is set to zero
  # There are 354 rows where 'CompetitionOpenSinceYear' and 'CompetitionOpenSinceMonth' is missing
  # Let's set these values to zeros 
  str_cols = ['Promo2SinceWeek', 'Promo2SinceYear', 'PromoInterval', 'CompetitionOpenSinceYear', 'CompetitionOpenSinceMonth']
  
  for str in str_cols:
      store_info_df [str].fillna(0, inplace = True)
  
  sns.heatmap(store_info_df.isnull(), yticklabels = False, cbar = False, cmap="Blues")
  
  # There are 3 rows with 'competitionDistance' values missing, let's fill them up with with average values of the 'CompetitionDistance' column
  store_info_df['CompetitionDistance'].fillna(store_info_df['CompetitionDistance'].mean(), inplace = True)
  
  sns.heatmap(store_info_df.isnull(), yticklabels = False, cbar = False, cmap="Blues")
  plt.savefig('after_handling_null_values.png')
  
  store_info_df.hist(bins = 30, figsize = (20,20), color = 'r');
  plt.savefig('store_infor_histplot.png')
  # half of stores are involved in promo 2
  # half of the stores have their competition at a distance of 0-3000m (3 kms away)
  
  """EXPLORING MERGED DATASET """
  
  # Let's merge both data frames together based on 'store'
  sales_train_all_df = pd.merge(sales_train_df, store_info_df, how = 'inner', on = 'Store')
  
  sales_train_all_df.to_csv('test.csv', index=False)
  
  sales_train_all_df
  
  correlations = sales_train_all_df.corr()['Sales'].sort_values()
  correlations
  
  # customers and promo are positively correlated with the sales 
  # Promo2 does not seem to be effective at all
  
  correlations = sales_train_all_df.corr()
  f, ax = plt.subplots(figsize = (20, 20))
  sns.heatmap(correlations, annot = True)
  plt.savefig('correaltion_among_features.png')
  # Customers/Prmo2 and sales are strongly correlated
  
  # Let's separate the year and put it into a separate column 
  sales_train_all_df['Year'] = pd.DatetimeIndex(sales_train_all_df['Date']).year
  
  sales_train_all_df
  
  # Let's do the same for the Day and Month
  sales_train_all_df['Month'] = pd.DatetimeIndex(sales_train_all_df['Date']).month
  sales_train_all_df['Day'] = pd.DatetimeIndex(sales_train_all_df['Date']).day
  
  sales_train_all_df
  
  # Let's take a look at the average sales and number of customers per month 
  # 'groupby' works great by grouping all the data that share the same month column, then obtain the mean of the sales column  
  # It looks like sales and number of customers peak around christmas timeframe
  axis = sales_train_all_df.groupby('Month')[['Sales']].mean().plot(figsize = (10,5), marker = 'o', color = 'r')
  axis.set_title('Average Sales Per Month')
  
  plt.figure()
  plt.savefig('avg_Sales_per_month.png')
  
  axis = sales_train_all_df.groupby('Month')[['Customers']].mean().plot(figsize = (10,5), marker = '^', color = 'b')
  axis.set_title('Average Customers Per Month')
  
  plt.savefig('avg_customer_per_month.png')
  
  # Let's take a look at the sales and customers per day of the month instead
  # Minimum number of customers are generally around the 24th of the month 
  # Most customers and sales are around 30th and 1st of the month
  ax = sales_train_all_df.groupby('Day')[['Sales']].mean().plot(figsize = (10,5), marker = 'o', color = 'r')
  axis.set_title('Average Sales Per Day')
  
  plt.figure()
  plt.savefig('avg_Sales_per_day.png')
  
  ax = sales_train_all_df.groupby('Day')[['Customers']].mean().plot(figsize = (10,5), marker = '^', color = 'b')
  axis.set_title('Average customer Per Day')
  plt.savefig('avg_customer_per_day.png')
  
  # Let's do the same for the day of the week  (note that 7 = Sunday)
  
  axis = sales_train_all_df.groupby('DayOfWeek')[['Sales']].mean().plot(figsize = (10,5), marker = 'o', color = 'r')
  axis.set_title('Average Sales Per Day of the Week')
  
  plt.figure()
  plt.savefig('avg_Sales_per_day_of_week.png')
  
  axis = sales_train_all_df.groupby('DayOfWeek')[['Customers']].mean().plot(figsize = (10,5), marker = '^', color = 'b')
  axis.set_title('Average Customers Per Day of the Week')
  plt.savefig('avg_customer_per_day_of_week.png')
  
  fig, ax = plt.subplots(figsize=(20,10))
  sales_train_all_df.groupby(['Date','StoreType']).mean()['Sales'].unstack().plot(ax=ax)
  plt.savefig('StoreType.png')
  
  plt.figure(figsize=[15,10])
  
  plt.subplot(211)
  sns.barplot(x = 'Promo', y = 'Sales', data = sales_train_all_df)
  plt.savefig('promo_vs_sales.png')
  
  plt.subplot(212)
  sns.barplot(x = 'Promo', y = 'Customers', data = sales_train_all_df)
  plt.savefig('promo_vs_customers.png')
  
  plt.figure(figsize=[15,10])
  
  plt.subplot(211)
  sns.violinplot(x = 'Promo', y = 'Sales', data = sales_train_all_df)
  plt.savefig('violinplot_promo_vs_sales.png')
  
  plt.subplot(212)
  sns.violinplot(x = 'Promo', y = 'Customers', data = sales_train_all_df)
  plt.savefig('violinplot_promo_vs_customers.png')
  
  """### TRAINING THE MODEL WITH FACEBOOK PROPHET
  
  for more info check out [link1](https://research.fb.com/prophet-forecasting-at-scale/), [link2](https://facebook.github.io/prophet/docs/quick_start.html#python-api)
  """
  
  !pip install fbprophet
  from fbprophet import Prophet
  
  def sales_prediction(Store_ID, sales_df, periods):
      
    # Function that takes in the data frame, storeID, and number of future period forecast
    # The function then generates date/sales columns in Prophet format
    # The function then makes time series predictions
      sales_df = sales_df[ sales_df['Store'] == Store_ID ]
      sales_df = sales_df[['Date', 'Sales']].rename(columns = {'Date': 'ds', 'Sales':'y'})
      sales_df = sales_df.sort_values('ds')
    
      model    = Prophet()
      model.fit(sales_df)
      future   = model.make_future_dataframe(periods=periods)
      forecast = model.predict(future)
      figure   = model.plot(forecast, xlabel='Date', ylabel='Sales')
      figure2  = model.plot_components(forecast)
  
  sales_prediction(10, sales_train_all_df, 60)
  
  """### TRAINING THE MODEL PART B
  
  - StateHoliday: indicates a state holiday. Normally all stores, with few exceptions, are closed on state holidays. Note that all schools are closed on public holidays and weekends. a = public holiday, b = Easter holiday, c = Christmas, 0 = None
     - SchoolHoliday: indicates if the (Store, Date) was affected by the closure of public schools
  """
  
  def sales_prediction(Store_ID, sales_df, holidays, periods):
    # Function that takes in the storeID and returns two date/sales columns in Prophet format
    # Format data to fit prophet 
  
    sales_df = sales_df[ sales_df['Store'] == Store_ID ]
    sales_df = sales_df[['Date', 'Sales']].rename(columns = {'Date': 'ds', 'Sales':'y'})
    sales_df = sales_df.sort_values('ds')
    
    model    = Prophet(holidays = holidays)
    model.fit(sales_df)
    future   = model.make_future_dataframe(periods = periods)
    forecast = model.predict(future)
    figure   = model.plot(forecast, xlabel='Date', ylabel='Sales')
    figure2  = model.plot_components(forecast)
  
  # Get all the dates pertaining to school holidays 
  school_holidays = sales_train_all_df[sales_train_all_df['SchoolHoliday'] == 1].loc[:, 'Date'].values
  school_holidays.shape
  
  school_holidays
  
  # Get all the dates pertaining to state holidays 
  state_holidays = sales_train_all_df [ (sales_train_all_df['StateHoliday'] == 'a') | (sales_train_all_df['StateHoliday'] == 'b') | (sales_train_all_df['StateHoliday'] == 'c')  ].loc[:, 'Date'].values
  state_holidays.shape
  
  state_holidays = pd.DataFrame({'ds': pd.to_datetime(state_holidays),
                                 'holiday': 'state_holiday'})
  
  state_holidays
  
  school_holidays = pd.DataFrame({'ds': pd.to_datetime(school_holidays),
                                  'holiday': 'school_holiday'})
  
  school_holidays
  
  # concatenate both school and state holidays 
  school_state_holidays = pd.concat((state_holidays, school_holidays))
  
  school_state_holidays
  
  # Let's make predictions using holidays for a specific store
  sales_prediction(6, sales_train_all_df, school_state_holidays, 60)