Visualizalising tabular data (Airbnb)¶

#disable some annoying warnings
import warnings
warnings.filterwarnings('ignore', category=FutureWarning)

#plots the figures in place instead of a new window
%matplotlib inline

import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

import pandas as pd
import numpy as np

reviews = pd.read_csv('reviews.csv.gz', parse_dates=['date'])
reviews.head()

fs, axs = plt.subplots(1, figsize=(15,10))
plt.title("Airbnb London: Number of reviews across time", fontsize=30)

reviews.groupby('date').aggregate({'listing_id':'count'}).reset_index().plot.scatter(x = 'date', y = 'listing_id', alpha = 0.25, ax = axs)
axs.set_ylabel('Number of reviews', fontsize=20)
axs.set_xlabel('Time', fontsize=20)
plt.show()

# getting the total number of reviews per day
reviews_time = reviews.groupby('date').aggregate({'listing_id':'count'}).reset_index()
reviews_time['year'] = reviews_time.date.dt.year

# Pre covid data
reviews_time_proper = reviews_time[(reviews_time.year < 2020)]

from statsmodels.tsa.seasonal import seasonal_decompose
result = seasonal_decompose(reviews_time_proper.listing_id, model='additive', period = 365)
print(result.plot())

y_values = result.trend[182:3136]
x_values = range(182, 3136)
coeffs = np.polyfit(x_values, y_values, 2)
poly_eqn = np.poly1d(coeffs)
y_hat = poly_eqn(range(365, len(reviews_time[reviews_time.year < 2020])))
y_hat1 = poly_eqn(range(len(reviews_time[reviews_time.year < 2020]), len(reviews_time)))

y_values = reviews_time[reviews_time.year == 2019].listing_id
x_values = range(365)
coeffs = np.polyfit(x_values, y_values, 15)
poly_eqn = np.poly1d(coeffs)
y_hat_seasonal = poly_eqn(range(365))

reviews_covid = reviews_time[reviews_time.year >= 2020]
y_values = reviews_covid.listing_id
x_values = range(len(reviews_covid.listing_id))
coeffs = np.polyfit(x_values, y_values, 17)
poly_eqn = np.poly1d(coeffs)
y_hat_covid = poly_eqn(range(25, len(reviews_covid.listing_id)-15))

from mpl_toolkits.axes_grid1.inset_locator import mark_inset, inset_axes
from matplotlib.patches import ConnectionPatch # making lines from top lot to below plot

# Two plots, the main on the top with height 20 inches and the bottom one is 10 inches.
fs, axs = plt.subplots(2, figsize=(20,30), gridspec_kw={'height_ratios': [2, 1]}, constrained_layout=True)

# Title
plt.suptitle("Airbnb London: Number of reviews across time", fontsize=30)

# First plot, main scatterplot
reviews.groupby('date').aggregate({'listing_id':'count'}).\
    reset_index().plot.scatter(x = 'date', y = 'listing_id', alpha = 0.25, ax = axs[0])

# adding the trend lines
axs[0].plot(reviews_time[365:len(reviews_time[reviews_time.year < 2020])].date,y_hat, color='red')
axs[0].plot(reviews_time[len(reviews_time[reviews_time.year < 2020]):].date,y_hat1, color='red', linestyle='dashed')

# Modifying the labels and title
axs[0].set_ylabel('Number of reviews', fontsize=15)
axs[0].set_xlabel('Time', fontsize=15)
axs[0].set_title('Total reviews of all types of rooms across London. A trend line is plotted taking the exponential growth of the business before Covid 19 and projecting the same trend during Covid. \n'+
'Seasonality before Covid is shown by zooming for 2019 (sample year). The affect of covid related lockdowns is also shown by zooming from 2020 onwards.',
                      fontsize=15, loc='left')

# Plotting the data within 2019 as a semantic zooming
axins = inset_axes(axs[0], 8, 5, loc=2, bbox_to_anchor=(0.15, 0.925),
                  bbox_transform=axs[0].figure.transFigure)

# Semantic zooming plot, scatterplot
reviews.groupby('date').aggregate({'listing_id':'count'}).reset_index().\
    plot.scatter(x = 'date', y = 'listing_id', alpha = 0.5, ax = axins)
#adding the trend lines, labels and title
plt.plot(reviews_time[reviews_time.year == 2019].date,y_hat_seasonal, color='red')
plt.ylabel('Number of reviews', fontsize=15)
plt.xlabel('Date', fontsize=15)
plt.title('Sesonality in a year (before COVID 19)', fontsize=20)

# Seasonality plot x and y limits
x1 = min(reviews_time[reviews_time.year == 2019].date)
x2 = max(reviews_time[reviews_time.year == 2019].date)
axins.set_xlim(x1, x2)
axins.set_ylim(0, 2000)
mark_inset(axs[0], axins, loc1=1, loc2=3, fc="none", ec="0.5")


# Second plot
x1 = min(reviews_time[reviews_time.year == 2020].date)
x2 = max(reviews_time[reviews_time.year >= 2020].date)

reviews_time[reviews_time.year >= 2020].plot.scatter(x = 'date', y = 'listing_id', alpha = 0.75, ax = axs[1])
axs[1].set_ylabel('Number of reviews', fontsize=15)
axs[1].set_xlabel('Date', fontsize=15)
axs[1].set_ylim(0, 1200)
axs[1].set_xlim(x1, x2)
axs[1].set_title('Effect of Covid19 on number of reviews', fontsize=20)

axs[1].plot(reviews_covid.date[25:-15],y_hat_covid, color='red')

# Adding annotations in the plot
axs[1].annotate(
        text = 'First Covid 19 advisory \n 3-16-2020', # the text
        xy=('3-16-2020', 500), #what to annotate
        xytext=('3-16-2020', 700), # where the text should be
        arrowprops=dict(arrowstyle="->",
                       connectionstyle="angle3,angleA=-90,angleB=0"),
        fontsize=15
    )
axs[1].annotate(
        'First Lockdown \n 3-23-2020', 
        xy=('3-23-2020', 350),
        xytext=('5-10-2020', 600), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Easing restrictions \n 7-4-2020', 
        xy=('7-4-2020', 50), 
        xytext=('6-15-2020', 700),
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Restrictions eased further \n 8-14-2020',
        xy=('8-14-2020', 250), 
        xytext=('8-1-2020', 600),
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Second Lockdown \n 10-31-2020', 
        xy=('10-31-2020', 165),
        xytext=('10-1-2020', 700), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Easing restrictions \n 12-2-2020', 
        xy=('12-2-2020', 120), 
        xytext=('11-10-2020', 600), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Christmas \n 12-25-2020',
        xy=('12-25-2020', 160), 
        xytext=('12-25-2020', 700), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Third Lockdown \n 1-6-2021',
        xy=('1-6-2021', 140),
        xytext=('1-20-2021', 600), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Schools reopen \n 3-8-2021', 
        xy=('3-8-2021', 125), 
        xytext=('2-25-2021', 700), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'All restrictions removed \n 6-21-2021', 
        xy=('6-21-2021', 400),
        xytext=('5-21-2021', 700), 
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )
axs[1].annotate(
        'Non essentials reopen \n 4-12-2021', 
        xy=('4-12-2021', 270), 
        xytext=('4-1-2021', 600),
        arrowprops=dict(arrowstyle="->"),
        fontsize=15
    )

# plotting the connections between the two plots
con = ConnectionPatch(xyA=(x1,-105), xyB=(x1,1200), coordsA="data", coordsB="data",
                      axesA=axs[0], axesB=axs[1])
axs[1].add_artist(con)
# con = ConnectionPatch(axesA=axs[0], axesB=axs[1])
con = ConnectionPatch(xyA=(x2,-105), xyB=(x2,1200), coordsA="data", coordsB="data",
                      axesA=axs[0], axesB=axs[1])
axs[1].add_artist(con)

plt.show()

listing_detailed = pd.read_csv('listings.csv.gz')

pd.options.display.max_columns = None # to show all the columns
listing_detailed.head()

listing_detailed.groupby('room_type').\
    aggregate({'number_of_reviews':'sum'}).\
    plot.pie(y='number_of_reviews', figsize=(10, 10), 
             autopct='%1.1f%%' # to add the percentages text
            )
plt.ylabel("")
plt.title("Airbnb London: Number of reviews by room type", fontsize=20)
plt.show()

# Function to combine last few classes ito one class
total_other_reviews = 0
def combine_last_n(df):
    df1 = df.copy()
    global total_other_reviews 
    total_other_reviews = (sum(df[df.number_of_reviews <= 5000].number_of_reviews))
    df1.loc[df.number_of_reviews <= 10000, 'number_of_reviews'] = 0
    return df1.number_of_reviews


listing_detailed.groupby('neighbourhood_cleansed').\
    aggregate({'number_of_reviews':'sum'}).\
    sort_values('number_of_reviews', ascending = False).\
    assign(no_reviews_alt = combine_last_n).\
    append(pd.Series({'number_of_reviews': 0, 'no_reviews_alt':total_other_reviews}, name='Others')).\
    plot.pie(y='no_reviews_alt', figsize=(10, 10), legend=None, rotatelabels=True, 
             wedgeprops=dict(width=.5) # for donut shape
            )
plt.ylabel("")

listings_sunburst = listing_detailed.groupby(['room_type', 'neighbourhood_cleansed']).\
    aggregate({'number_of_reviews':'sum'}).\
    sort_values('number_of_reviews', ascending = False).reset_index()

fig = px.sunburst(listings_sunburst, path=['room_type', 'neighbourhood_cleansed'], values='number_of_reviews',
                  hover_data = ['room_type', 'number_of_reviews'], hover_name = 'neighbourhood_cleansed',
                 title = 'Airbnb London: Popularity sunburst chart', width = 900, height = 900)
fig.show()

top_20_names = list(listings_sunburst.sort_values('number_of_reviews', ascending = False).head(30)['neighbourhood_cleansed'])

listings_sunburst_wide = listings_sunburst.\
    pivot(index='neighbourhood_cleansed', columns='room_type', values='number_of_reviews').reset_index()
listings_sunburst_wide = listings_sunburst_wide.replace(np.nan, 0)

fig = px.parallel_coordinates(
    listings_sunburst_wide
)

# add the pink line to highlight Kensington
fig.data[0]['dimensions'][0]['constraintrange'] = [50000, 60000]

fig.update_layout(title_text='Kensington and Chelsea (in blue) popularity ranking across room types', title_x=0.7, title_y = 0.05)

fig.show()

reviews_detailed = pd.merge(listing_detailed[['id', 'neighbourhood_cleansed', 'room_type']], reviews, left_on = 'id', right_on = 'listing_id')

fs, axs = plt.subplots(1, figsize=(15,10))
reviews_detailed['year'] = reviews_detailed.date.dt.year
reviews_detailed.groupby(['year', 'room_type']).\
    aggregate({'listing_id':'count'}).\
    unstack().reset_index().\
    plot.bar(x = 'year', y = 'listing_id', ax = axs, stacked = True)
plt.title('Popularity of different rooms across the years', fontsize = 25)
plt.legend(loc='upper right', title="Type of room", fontsize='medium',fancybox=True)
axs.set_ylabel('Number of reviews', fontsize=20)
axs.set_xlabel('Years', fontsize=20)

plt.show()

fs, ax = plt.subplots(1, figsize=(15,10))
reviews_room = reviews_detailed.groupby(['date', 'room_type']).\
    aggregate({'listing_id':'count'}).\
    unstack().reset_index()
reviews_room.columns = ['date', 'Entire home/apt', 'Hotel room', 'Private room', 'Shared room']
ax.stackplot(reviews_room.date, 
            list(reviews_room[['Entire home/apt', 'Hotel room', 'Private room', 'Shared room']].fillna(0).\
                 to_numpy().transpose()),
            baseline='wiggle')
plt.title('Popularity of different rooms across the years', fontsize=20, y=1.05,loc='left')
ax.text("2010",1050,'Streamgraph of the number of reviews across time',ha='left', fontsize=12)

plt.legend(['Entire home/apt', 'Hotel room', 'Private room', 'Shared room'], loc='upper left', title="Type of room")
ax.set_xlabel('Years', fontsize=20)

plt.show()

location_rating = listing_detailed.groupby(['neighbourhood_cleansed', 'room_type']).\
    aggregate({'review_scores_rating':'mean'}).unstack().reset_index()
location_rating.columns = ['neighbourhood', 'Entire home/apt', 'Hotel room', 'Private room', 'Shared room']
location_rating.index = location_rating.neighbourhood

fig, ax = plt.subplots(figsize=[20,len(location_rating)/3.3])
sns.heatmap(data=location_rating[['Entire home/apt', 'Hotel room', 'Private room', 'Shared room']],
            annot=False, cbar_kws={"shrink":0.5,"orientation":'vertical'},linewidths=0.004,linecolor='grey',vmin=0,vmax=5, center = 0.25)

plt.show()

def add_regions(df, borough_col_name):
    """
    This function takes as input a dataframe with a column which includes London's borough name
    Then returns the same dataframw with sub regions names added for each borough
    """
    central = ['Camden', 'City of London', 'Kensington and Chelsea', 'Islington', 'Lambeth', 'Southwark', 'Westminster']
    east = ['Barking and Dagenham', 'Bexley', 'Greenwich', 'Hackney', 'Havering', 'Lewisham', 'Newham', 'Redbridge', 'Tower Hamlets', 'Waltham Forest']
    north = ['Barnet', 'Enfield', 'Haringey']
    south = ['Bromley', 'Croydon', 'Kingston upon Thames', 'Merton', 'Sutton', 'Wandsworth']
    west = ['Brent', 'Ealing', 'Hammersmith and Fulham', 'Harrow', 'Richmond upon Thames', 'Hillingdon', 'Hounslow']


    df['sub_regions'] = np.where((df[borough_col_name] == central[0]) | 
                                       (df[borough_col_name] == central[1]) |
                                       (df[borough_col_name] == central[2]) |
                                       (df[borough_col_name] == central[3]) |
                                       (df[borough_col_name] == central[4]) |
                                       (df[borough_col_name] == central[5]) |
                                       (df[borough_col_name] == central[6]) 
                                       , 'Central', 'no')

    df['sub_regions'] = np.where((df[borough_col_name] == east[0]) | 
                                       (df[borough_col_name] == east[1]) |
                                       (df[borough_col_name] == east[2]) |
                                       (df[borough_col_name] == east[3]) |
                                       (df[borough_col_name] == east[4]) |
                                       (df[borough_col_name] == east[5]) |
                                       (df[borough_col_name] == east[6]) |
                                       (df[borough_col_name] == east[7]) |
                                       (df[borough_col_name] == east[8]) |
                                       (df[borough_col_name] == east[9]) 
                                       , 'East', df['sub_regions'])

    df['sub_regions'] = np.where((df[borough_col_name] == north[0]) | 
                                       (df[borough_col_name] == north[1]) |
                                       (df[borough_col_name] == north[2]) 
                                       , 'North', df['sub_regions'])

    df['sub_regions'] = np.where((df[borough_col_name] == south[0]) | 
                                       (df[borough_col_name] == south[1]) | 
                                       (df[borough_col_name] == south[2]) | 
                                       (df[borough_col_name] == south[3]) | 
                                       (df[borough_col_name] == south[4]) | 
                                       (df[borough_col_name] == south[5])  
                                       , 'South', df['sub_regions'])

    df['sub_regions'] = np.where((df[borough_col_name] == west[0]) | 
                                       (df[borough_col_name] == west[1]) | 
                                       (df[borough_col_name] == west[2]) | 
                                       (df[borough_col_name] == west[3]) | 
                                       (df[borough_col_name] == west[4]) | 
                                       (df[borough_col_name] == west[5]) | 
                                       (df[borough_col_name] == west[6]) 
                                       , 'West', df['sub_regions'])

    return df
def sort_data(df):
    """
    Groups the data by location and sorts the data based on average rating within the location. 
    Different locations are sorted by average rating.
    """
    df1 = df.copy()
    df1['average_rating'] = (df1['Entire home/apt']+ df1['Hotel room'] + df1['Private room']+df1['Shared room'])/4
    df1['location_average'] = df1.groupby('sub_regions')['average_rating'].transform('mean')
    df1 = df1.sort_values(['location_average', 'average_rating'], ascending=False)
    return df1[['Private room', 'Entire home/apt', 'Hotel room', 'Shared room', 'sub_regions']]

def prepare_reg_annotation_lists():
    """
    Creates the annotation in the form of groups within the data. Displays this on the right hand side of the heatmap.
    """
    reg_sorted_list = location_rating.sub_regions.unique()
    reg_len = location_rating.sub_regions.value_counts().to_dict()

    sorted_len = []
    cum_len = []
    arrow_style_str_list = []

    # here we define the width of the bracket used, which is proportional to the number of boroughs within a sub region
    for i in range(5):
        if i==0:
            value = reg_len[reg_sorted_list[i]]
            cum_value = value
        else:
            value = reg_len[reg_sorted_list[i]]
            cum_value += value

        sorted_len.append(value)
        cum_len.append(cum_value)
        arrow_style_str_list.append('-[,widthB='+str((value/1.2)-0.5)+',lengthB=0.7')

    # here we define ticks which represent the center location of each sub region relative to the heatmap
    Ticks = []
    for i in range(5):
        if i == 0:
            Ticks.append(1-((cum_len[i]/2)/33))
        else:
            Ticks.append(1-((((cum_len[i]-cum_len[i-1])/2)+cum_len[i-1])/33))

    return Ticks,arrow_style_str_list,reg_sorted_list


location_rating = location_rating.replace(np.nan, 2.5)
location_rating = add_regions(location_rating, 'neighbourhood')
location_rating = sort_data(location_rating)
Ticks_h,arrow,region = prepare_reg_annotation_lists()

red_green_cmap = sns.diverging_palette(10, 133,as_cmap=True)
red_green_cmap

fig, ax = plt.subplots(figsize=[20,len(location_rating)/3.3])
sns.heatmap(data=location_rating[['Private room', 'Entire home/apt', 'Hotel room', 'Shared room']],
            annot=False, cbar_kws={"shrink":0.5,"orientation":'vertical'},linewidths=0.004,linecolor='grey',vmin=2.25,vmax=4.75,
           cmap = red_green_cmap)
plt.title("Average ratings for different locations in London", fontsize=20, y=1.1,loc='left')
plt.text(0,-1,'Heatmap depicting the ratings among different locations in London. If no rating is available, minimum rating of 2.5 is assumed. \n Good ratings are ratings above 3.5 while bad ratings are below. The data is grouped by location (right) and sorted by average rating.',ha='left', fontsize=12)
ax.set_ylabel('')

#annotation for the borough
for i in range(5):
    ax.annotate(region[i],xy=(1.01,Ticks_h[i]), xytext=(1.02,Ticks_h[i]), xycoords='axes fraction',
                ha='left',va='center', arrowprops=dict(arrowstyle=arrow[i],lw=1))       

reviews_treemap = listing_detailed.groupby(['neighbourhood_cleansed']).\
    aggregate({'review_scores_rating':'mean', 'number_of_reviews':'sum'}).reset_index()
reviews_treemap = add_regions(reviews_treemap, 'neighbourhood_cleansed')

#change col names for nice viz on hover
reviews_treemap.columns = ['neighbourhood', 'Average Reviews', 'Number of reviews', 'regions']

fig = px.treemap(reviews_treemap, path=[px.Constant("London"), 'regions', 'neighbourhood'], values='Number of reviews', 
           color = 'Average Reviews', color_continuous_scale='RdBu')
fig.update_layout(margin = dict(t=50, l=25, r=25, b=25))
fig.update_layout(title_text='Airbnb London: Ratings overview')

reviews_detailed_text = pd.merge(listing_detailed[['id', 'description','neighborhood_overview', 'host_about', 'review_scores_rating']], reviews, left_on = 'id', right_on = 'listing_id')
reviews_detailed_positive_text = reviews_detailed_text[reviews_detailed_text.review_scores_rating > 3.75]
reviews_detailed_negative_text = reviews_detailed_text[reviews_detailed_text.review_scores_rating <= 3.75]

pos_reviews_text = reviews_detailed_positive_text.sample(n=100, random_state = 2).comments.str.cat()
neg_reviews_text = reviews_detailed_negative_text.sample(n=100, random_state = 3).comments.str.cat()

# !pip install wordcloud
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from PIL import Image
mask_pos = np.array(Image.open('thumbs-up-xxl.png'))

# word cloud, good vs bad ratings
stop_words = ["https", "co", "RT", 'br', '<br>', '<br/>', '\r', 'r'] + list(STOPWORDS)
wordcloud_pattern = WordCloud(stopwords = stop_words, background_color="white", max_words=2000, max_font_size=256,
               random_state=42, mask = mask_pos, width=mask_pos.shape[1], height=mask_pos.shape[0])
wordcloud_positive = wordcloud_pattern.generate(pos_reviews_text)

plt.imshow(wordcloud_positive, interpolation='bilinear')
plt.axis("off")
plt.show()

mask_neg = np.array(Image.open('thumbs-down-xxl.png'))

# word cloud, good vs bad ratings
wordcloud_pattern = WordCloud(stopwords = stop_words, background_color="white", max_words=2000, max_font_size=256,
               random_state=42, mask = mask_neg, width=mask_neg.shape[1], height=mask_neg.shape[0])
wordcloud_neg = wordcloud_pattern.generate(neg_reviews_text)

plt.imshow(wordcloud_neg, interpolation='bilinear')
plt.axis("off")
plt.show()

fs, axs = plt.subplots(1, 2, figsize=(20,10))
plt.suptitle("Airbnb London: Wordcloud of positive and negative reviews", fontsize=20)
plt.figtext(0.5, 0.925, 'Wordcloud derived from a random sample of 100 positive and 100 negative reviews.',
            wrap=True, horizontalalignment='center', fontsize=12)

axs[0].imshow(wordcloud_positive, interpolation='bilinear')
axs[0].axis("off")

axs[1].imshow(wordcloud_neg, interpolation='bilinear')
axs[1].axis("off")

plt.show()

listing_detailed['host_response_rate'] = listing_detailed['host_response_rate'].str.replace('%', '').astype(float)
listing_detailed['host_acceptance_rate'] = listing_detailed['host_acceptance_rate'].str.replace('%', '').astype(float)
listing_detailed['price'] = listing_detailed['price'].str.replace('$', '').str.replace(',', '').astype(float)

col_for_corr = ['review_scores_rating', 'review_scores_accuracy', 'review_scores_cleanliness', 'review_scores_checkin', 'review_scores_communication', 'review_scores_location', 'review_scores_value',
                'number_of_reviews', 'number_of_reviews_ltm', 'number_of_reviews_l30d', 'reviews_per_month',
                'availability_30', 'availability_60', 'availability_90', 'availability_365',
                'minimum_nights', 'maximum_nights', 'minimum_minimum_nights', 'maximum_minimum_nights', 'minimum_maximum_nights', 'maximum_maximum_nights', 'minimum_nights_avg_ntm', 'maximum_nights_avg_ntm',
                'bedrooms', 'beds','accommodates', 'price', 
                'host_response_rate', 'host_acceptance_rate', 'host_total_listings_count']


f = plt.figure(figsize=(20, 20))
plt.matshow(listing_detailed[col_for_corr].corr(), fignum = f, cmap = red_green_cmap, vmin=-1, vmax=1)
plt.xticks(range(listing_detailed[col_for_corr].select_dtypes(['number']).shape[1]), 
           listing_detailed[col_for_corr].select_dtypes(['number']).columns, rotation=90, fontsize = 15)
plt.yticks(range(listing_detailed[col_for_corr].select_dtypes(['number']).shape[1]), 
           listing_detailed[col_for_corr].select_dtypes(['number']).columns, rotation=0, fontsize = 15)
cb = plt.colorbar()
cb.ax.tick_params(labelsize=14)
plt.title("Airbnb London: Correlation between different parameters", fontsize=20,loc='left')
plt.text(0,32,'Correlation matrix displaying the different parameters within the data. DIverging green (+ve) red (-ve) scale is used to display the correlations.\n \
    Features considered: Review scores, Number of reviews, availability, maximum and minimum days of stay, host and room parameters.',
         ha='left', fontsize=12)

plt.show()

def annotate_plot(x, y, **kwargs):

    if(x.name == 'price' and y.name == 'review_scores_rating'):
        ax = plt.gca()
        for index, obj in listing_detailed.nlargest(2, 'price').iterrows():
            plt.annotate(
                obj['name'], # the text
                xy=(obj.price, obj.review_scores_rating), 
                xytext=(7500, obj.review_scores_rating-0.5),
                arrowprops=dict(arrowstyle="->") 
            )
    elif(x.name == 'price' and y.name == 'number_of_reviews'):
        ax = plt.gca()
        for index, obj in listing_detailed.nlargest(3, 'number_of_reviews').iterrows():
            ax.text(obj.price, obj.number_of_reviews, obj['name'])

col_for_pairplot = ['review_scores_rating', 'number_of_reviews',  'price']

sns_plot = sns.pairplot(listing_detailed, vars = col_for_pairplot,  kind='scatter',  hue = 'room_type', diag_kind='kde',)
sns_plot.fig.set_size_inches(20, 20)
sns_plot._legend.set_bbox_to_anchor((0.15, 0.89))
sns_plot.map_upper(annotate_plot)
sns_plot.fig.suptitle("Airbnb London: Scatterplot matrix", fontsize = 20, y=1)

fs, axs = plt.subplots(1, 2, figsize=(20,20))
listing_detailed.boxplot(column = 'review_scores_rating', by = 'room_type', figsize = (10,20), ax = axs[0])
sns.violinplot('room_type','review_scores_rating', data=listing_detailed, ax = axs[1])
plt.suptitle("Airbnb London: Average rating across room types", fontsize = 20, y=0.95)

plt.figtext(0.5, 0.925, 'Boxplot (left) and Violin plot (right) for the average review across room types',
            wrap=True, horizontalalignment='center', fontsize=12)

axs[0].set_title('')

for ax in axs:
    ax.set_ylim(-1, 6)
    ax.set_ylabel('Average Rating', fontsize=12)
    ax.set_xlabel('Room types', fontsize=12)

from sklearn.preprocessing import MinMaxScaler
import seaborn as sns
col_for_corr = [ 'price', 'review_scores_rating', 'number_of_reviews', 'availability_90', 'minimum_nights_avg_ntm', 'bedrooms', 
                'host_response_rate']

df_cluster = listing_detailed.groupby('neighbourhood_cleansed').aggregate({
    'review_scores_rating':'mean',
    'number_of_reviews':'sum',
    'availability_90':'mean',
    'minimum_nights_avg_ntm':'mean',
    'bedrooms':'mean',
    'price':'mean', 
    'host_response_rate':'mean'
}).reset_index()

scaler = MinMaxScaler()
df_cluster1 = pd.DataFrame(scaler.fit_transform(df_cluster[col_for_corr]), columns=col_for_corr)
df_cluster1.index = df_cluster.neighbourhood_cleansed

crest_cmap = sns.color_palette("crest", as_cmap=True)
crest_cmap

g = sns.clustermap(df_cluster1, cmap = crest_cmap, vmin = 0, vmax = 1)
plt.title("Airbnb London: Clusters within London", fontsize=20,loc='left', y = 2, x = -25)
g.ax_cbar.set_position((1, .2, .03, .4))
g.ax_heatmap.set_ylabel("")

plt.show()