In [38]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import glob
from scipy import stats

%matplotlib inline  
sns.set(rc={"figure.figsize": (14, 8)})
sns.set_palette('muted')

Parsowanie logów

In [39]:
fnPattern = 'excento/logs-AM8/*.s'
fns = glob.glob(fnPattern)
print(len(fns))
93
In [40]:
sensors_expected = ['PMS_5003', 'PMS_7003', 'PMS_A003', 'SDS_021', 'ANALOG']
dfs = []
sensors = {}
j = 0
for fn in fns:
    j += 1
    prct = '{:.2f}'.format(j / len(fns) * 100.0)
    print(j, prct, fn)
    i = 0
    data = {}
    with open(fn) as file:
        for line in file:
            try:
                sens = line.split(' ')[3].replace(':', '')
                if sens in sensors_expected:
                    datetime = ' '.join(line.split(' ')[1:3])[0:-4]
                    datadict = line.split('|')[1].split()
                    d = dict(zip(datadict[::2], list(map(float, datadict[1::2]))))
                    data[(datetime, sens)] = d
                    sensors[sens] = d.keys()
                else:
                    print('_unexpected name:', sens)
            except IndexError:
                print('_error in index:', fn, i, '--->', line)
            except:
                print('_error', fn, i, '--->', line)
                 
            i += 1
#             if (i % 2000 == 0):
#                 print('----', i)
                    
        print('\t try create DataFrame for', fn)
        df = pd.DataFrame(data).stack().swaplevel().sort_index().transpose()
        df.index = pd.to_datetime(df.index)
        dfs.append(df)
                
1 1.08 excento/logs-AM8/@400000005a7aabc022707f70.s
	 try create DataFrame for excento/logs-AM8/@400000005a7aabc022707f70.s
2 2.15 excento/logs-AM8/@400000005a7b25201c0533d8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b25201c0533d8.s
3 3.23 excento/logs-AM8/@400000005a79c36701bac790.s
	 try create DataFrame for excento/logs-AM8/@400000005a79c36701bac790.s
4 4.30 excento/logs-AM8/@400000005a7b16c50fc65098.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b16c50fc65098.s
5 5.38 excento/logs-AM8/@400000005a7b414618d10318.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b414618d10318.s
6 6.45 excento/logs-AM8/@400000005a7a63be13eb6168.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a63be13eb6168.s
7 7.53 excento/logs-AM8/@400000005a7b94a71bf26b40.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b94a71bf26b40.s
8 8.60 excento/logs-AM8/@400000005a7a9657352fa960.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a9657352fa960.s
9 9.68 excento/logs-AM8/@400000005a7a221239f1ffe8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a221239f1ffe8.s
10 10.75 excento/logs-AM8/@400000005a7b0858310cbf58.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b0858310cbf58.s
11 11.83 excento/logs-AM8/@400000005a7c127a308b0468.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c127a308b0468.s
12 12.90 excento/logs-AM8/@400000005a7c19912c415880.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c19912c415880.s
13 13.98 excento/logs-AM8/@400000005a7b9b952baeada0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b9b952baeada0.s
14 15.05 excento/logs-AM8/@400000005a7a87f601583530.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a87f601583530.s
15 16.13 excento/logs-AM8/@400000005a7a7248059aad58.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a7248059aad58.s
16 17.20 excento/logs-AM8/@400000005a7a138230aa05e8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a138230aa05e8.s
17 18.28 excento/logs-AM8/@400000005a7b0f900c3532c8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b0f900c3532c8.s
18 19.35 excento/logs-AM8/@400000005a7b63f70a009ba0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b63f70a009ba0.s
19 20.43 excento/logs-AM8/@400000005a7bcc5a02921c90.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bcc5a02921c90.s
20 21.51 excento/logs-AM8/@400000005a79a5d31a3ccc00.s
	 try create DataFrame for excento/logs-AM8/@400000005a79a5d31a3ccc00.s
21 22.58 excento/logs-AM8/@400000005a7ab2d62e6f2178.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ab2d62e6f2178.s
22 23.66 excento/logs-AM8/@400000005a79e0850b892000.s
	 try create DataFrame for excento/logs-AM8/@400000005a79e0850b892000.s
23 24.73 excento/logs-AM8/@400000005a7bbe59145e5b00.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bbe59145e5b00.s
24 25.81 excento/logs-AM8/@400000005a7c04931db3d608.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c04931db3d608.s
25 26.88 excento/logs-AM8/@400000005a7a37f7184f3888.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a37f7184f3888.s
26 27.96 excento/logs-AM8/@400000005a7acef10f789628.s
	 try create DataFrame for excento/logs-AM8/@400000005a7acef10f789628.s
27 29.03 excento/logs-AM8/@400000005a7b7fe237dcfeb0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b7fe237dcfeb0.s
28 30.11 excento/logs-AM8/@400000005a7bfda406d61770.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bfda406d61770.s
29 31.18 excento/logs-AM8/@400000005a79f65a020b6448.s
	 try create DataFrame for excento/logs-AM8/@400000005a79f65a020b6448.s
30 32.26 excento/logs-AM8/@400000005a7add2c3a2d5368.s
	 try create DataFrame for excento/logs-AM8/@400000005a7add2c3a2d5368.s
31 33.33 excento/logs-AM8/@400000005a79d20904dcf308.s
	 try create DataFrame for excento/logs-AM8/@400000005a79d20904dcf308.s
32 34.41 excento/logs-AM8/@400000005a7ae4632aaa5918.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ae4632aaa5918.s
33 35.48 excento/logs-AM8/@400000005a79ef0e0ad62f68.s
	 try create DataFrame for excento/logs-AM8/@400000005a79ef0e0ad62f68.s
34 36.56 excento/logs-AM8/@400000005a7a552d267f29e0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a552d267f29e0.s
35 37.63 excento/logs-AM8/@400000005a7b483210876e18.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b483210876e18.s
36 38.71 excento/logs-AM8/@400000005a7b2c410e3c5920.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b2c410e3c5920.s
37 39.78 excento/logs-AM8/@400000005a7b3a5b28eb5f50.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b3a5b28eb5f50.s
38 40.86 excento/logs-AM8/@400000005a79cabc35537958.s
	 try create DataFrame for excento/logs-AM8/@400000005a79cabc35537958.s
39 41.94 excento/logs-AM8/@400000005a7a04f12c78eb10.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a04f12c78eb10.s
40 43.01 excento/logs-AM8/@400000005a7be8820bb200d8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7be8820bb200d8.s
41 44.09 excento/logs-AM8/@400000005a7b012225ddf070.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b012225ddf070.s
42 45.16 excento/logs-AM8/@400000005a7b86d13072fc10.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b86d13072fc10.s
43 46.24 excento/logs-AM8/@400000005a7b56131c75b4a0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b56131c75b4a0.s
44 47.31 excento/logs-AM8/@400000005a7af9f331bb56a8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7af9f331bb56a8.s
45 48.39 excento/logs-AM8/@400000005a7bda660ffd3b30.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bda660ffd3b30.s
46 49.46 excento/logs-AM8/@400000005a7ad6080e7d55d8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ad6080e7d55d8.s
47 50.54 excento/logs-AM8/@400000005a7aeb91035f8680.s
	 try create DataFrame for excento/logs-AM8/@400000005a7aeb91035f8680.s
48 51.61 excento/logs-AM8/@400000005a7bb06b367cf598.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bb06b367cf598.s
49 52.69 excento/logs-AM8/@400000005a7c0b840c557498.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c0b840c557498.s
50 53.76 excento/logs-AM8/@400000005a7b4f1e174d35c0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b4f1e174d35c0.s
51 54.84 excento/logs-AM8/@400000005a7996f336808b90.s
_unexpected name: exists
_unexpected name: |21

_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 2 ---> |

_unexpected name: **********************************
_unexpected name: **********************************
_unexpected name: Thread
_unexpected name: API
_unexpected name: API
_unexpected name: API
_unexpected name: TERMO_BUS
_unexpected name: open
_unexpected name: Error
_unexpected name: Error
_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 13 ---> Receive error

_unexpected name: 
_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 15 ---> "error "Receive error""

_unexpected name: 
_unexpected name: 
_unexpected name: 0x20
_unexpected name: Error
_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 386 ---> receive: timeout

_unexpected name: 
_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 388 ---> "error "receive: $::rh($fd,status)""

_unexpected name: 
_unexpected name: 
_error in index: excento/logs-AM8/@400000005a7996f336808b90.s 391 ---> "receive $fd $rlen"

_unexpected name: 
_unexpected name: 
_unexpected name: 0x20
	 try create DataFrame for excento/logs-AM8/@400000005a7996f336808b90.s
52 55.91 excento/logs-AM8/@400000005a79fda900a9d878.s
	 try create DataFrame for excento/logs-AM8/@400000005a79fda900a9d878.s
53 56.99 excento/logs-AM8/@400000005a7b78e312031968.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b78e312031968.s
54 58.06 excento/logs-AM8/@400000005a7c280f2b44a130.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c280f2b44a130.s
55 59.14 excento/logs-AM8/@400000005a79e7c834aab638.s
	 try create DataFrame for excento/logs-AM8/@400000005a79e7c834aab638.s
56 60.22 excento/logs-AM8/@400000005a7b335819ed40b8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b335819ed40b8.s
57 61.29 excento/logs-AM8/@400000005a7ba2850e7b17a0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ba2850e7b17a0.s
58 62.37 excento/logs-AM8/@400000005a7a295d127cae68.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a295d127cae68.s
59 63.44 excento/logs-AM8/@400000005a7b71e62334ae18.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b71e62334ae18.s
60 64.52 excento/logs-AM8/@400000005a7a30aa1d6a5988.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a30aa1d6a5988.s
61 65.59 excento/logs-AM8/@400000005a79d94908da6f08.s
	 try create DataFrame for excento/logs-AM8/@400000005a79d94908da6f08.s
62 66.67 excento/logs-AM8/@400000005a7bf6a637b4f898.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bf6a637b4f898.s
63 67.74 excento/logs-AM8/@400000005a7c20ce1900ecb8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7c20ce1900ecb8.s
64 68.82 excento/logs-AM8/@400000005a7bb75f21e86798.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bb75f21e86798.s
65 69.89 excento/logs-AM8/@400000005a7a0c39364347f8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a0c39364347f8.s
66 70.97 excento/logs-AM8/@400000005a7b5d06204a47f8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b5d06204a47f8.s
67 72.04 excento/logs-AM8/@400000005a7a3f4615f7f278.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a3f4615f7f278.s
68 73.12 excento/logs-AM8/@400000005a7ac7ec2fb53dd8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ac7ec2fb53dd8.s
69 74.19 excento/logs-AM8/@400000005a7bc5590b56c560.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bc5590b56c560.s
70 75.27 excento/logs-AM8/@400000005a7a4de037d4d4d8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a4de037d4d4d8.s
71 76.34 excento/logs-AM8/@400000005a7a8f2920562ed8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a8f2920562ed8.s
72 77.42 excento/logs-AM8/@400000005a7a6b03274469f8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a6b03274469f8.s
73 78.49 excento/logs-AM8/@400000005a7be171187b8078.s
	 try create DataFrame for excento/logs-AM8/@400000005a7be171187b8078.s
74 79.57 excento/logs-AM8/@400000005a7ba9772d9a0ab0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ba9772d9a0ab0.s
75 80.65 excento/logs-AM8/@400000005a7b1df51ba312c0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b1df51ba312c0.s
76 81.72 excento/logs-AM8/@400000005a7ab9ee1b21ec40.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ab9ee1b21ec40.s
77 82.80 excento/logs-AM8/@400000005a7a9d820542e898.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a9d820542e898.s
78 83.87 excento/logs-AM8/@400000005a79b49c37c3a280.s
	 try create DataFrame for excento/logs-AM8/@400000005a79b49c37c3a280.s
79 84.95 excento/logs-AM8/@400000005a7b8dbc34dcc6a0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b8dbc34dcc6a0.s
80 86.02 excento/logs-AM8/@400000005a79ad3d28c11ee8.s
	 try create DataFrame for excento/logs-AM8/@400000005a79ad3d28c11ee8.s
81 87.10 excento/logs-AM8/@400000005a79bc010a823b38.s
	 try create DataFrame for excento/logs-AM8/@400000005a79bc010a823b38.s
82 88.17 excento/logs-AM8/@400000005a7bef960c8d0b10.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bef960c8d0b10.s
83 89.25 excento/logs-AM8/@400000005a7af2c50e971f68.s
	 try create DataFrame for excento/logs-AM8/@400000005a7af2c50e971f68.s
84 90.32 excento/logs-AM8/@400000005a7a46940a26ffc0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a46940a26ffc0.s
85 91.40 excento/logs-AM8/@400000005a7a1ac907777fc0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a1ac907777fc0.s
86 92.47 excento/logs-AM8/@400000005a7b6aee10988ce8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7b6aee10988ce8.s
87 93.55 excento/logs-AM8/@400000005a7aa4a4294cc8d0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7aa4a4294cc8d0.s
88 94.62 excento/logs-AM8/@400000005a7a80c00e902258.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a80c00e902258.s
89 95.70 excento/logs-AM8/@400000005a7bd36130105df8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7bd36130105df8.s
90 96.77 excento/logs-AM8/@400000005a799e5f1ed3ec08.s
	 try create DataFrame for excento/logs-AM8/@400000005a799e5f1ed3ec08.s
91 97.85 excento/logs-AM8/@400000005a7a5c780a32aff0.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a5c780a32aff0.s
92 98.92 excento/logs-AM8/@400000005a7ac0e91e5117d8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7ac0e91e5117d8.s
93 100.00 excento/logs-AM8/@400000005a7a79850b4a07a8.s
	 try create DataFrame for excento/logs-AM8/@400000005a7a79850b4a07a8.s

Sprawdzenie kanałów pomiarowych

In [41]:
for k,v in sensors.items():
    print(k, ':\t', list(v))
PMS_5003 :	 ['PM01S', 'PM25S', 'PM10S', 'PM01A', 'PM25A', 'PM10A', 'PM0o3', 'PM0o5', 'PM1o0', 'PM2o5', 'PM5o0', 'PM10o']
ANALOG :	 ['PPD42-OUT1', 'PPD42-OUT2', 'AMPHENOL-OUT1', 'AMPHENOL-OUT2']
PMS_A003 :	 ['PM01S', 'PM25S', 'PM10S', 'PM01A', 'PM25A', 'PM10A', 'PM0o3', 'PM0o5', 'PM1o0', 'PM2o5', 'PM5o0', 'PM10o']
PMS_7003 :	 ['PM01S', 'PM25S', 'PM10S', 'PM01A', 'PM25A', 'PM10A', 'PM0o3', 'PM0o5', 'PM1o0', 'PM2o5', 'PM5o0', 'PM10o']
SDS_021 :	 ['PM25', 'PM10']

Ustawienie strefy czasowej i zapis całości

In [42]:
pm = pd.concat(dfs)
pm = pm.sort_index()
pm = pm.tz_localize('UTC')
pm = pm.tz_convert('Europe/Warsaw')
pm = pm.loc['2018-02-06 14:00':,:]
In [43]:
pm.to_pickle('pm10.pkl')
# pm = pd.read_pickle('pm10.pkl')

Sprawdzenie początku i końca tabeli

In [44]:
print(pm.shape)
pd.set_option('display.max_columns',100)
pm.head()
(150895, 42)
Out[44]:
ANALOG PMS_5003 PMS_7003 PMS_A003 SDS_021
AMPHENOL-OUT1 AMPHENOL-OUT2 PPD42-OUT1 PPD42-OUT2 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM10 PM25
2018-02-06 14:00:00+01:00 0.99883 1.0 1.0 1.0 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
2018-02-06 14:00:01+01:00 0.99883 1.0 1.0 1.0 8.0 8.0 924.0 274.0 12.0 12.0 1.0 33.0 9.0 9.0 5.0 3.0 5.0 5.0 966.0 296.0 8.0 8.0 0.0 46.0 8.0 8.0 4.0 0.0 9.0 9.0 1155.0 379.0 13.0 13.0 0.0 54.0 13.0 13.0 2.0 0.0 8.5 6.4
2018-02-06 14:00:02+01:00 0.99883 1.0 1.0 1.0 8.0 8.0 924.0 274.0 12.0 12.0 1.0 33.0 9.0 9.0 5.0 3.0 5.0 5.0 981.0 296.0 8.0 8.0 0.0 46.0 8.0 8.0 4.0 0.0 9.0 9.0 1167.0 385.0 13.0 13.0 0.0 56.0 13.0 13.0 0.0 0.0 8.5 6.4
2018-02-06 14:00:03+01:00 0.99883 1.0 1.0 1.0 8.0 8.0 954.0 284.0 14.0 14.0 1.0 41.0 11.0 11.0 5.0 3.0 5.0 5.0 981.0 296.0 8.0 8.0 0.0 46.0 8.0 8.0 4.0 0.0 9.0 9.0 1215.0 396.0 13.0 13.0 0.0 61.0 13.0 13.0 0.0 0.0 8.4 6.4
2018-02-06 14:00:04+01:00 0.99883 1.0 1.0 1.0 8.0 8.0 954.0 284.0 14.0 14.0 1.0 41.0 11.0 11.0 5.0 3.0 5.0 5.0 981.0 296.0 8.0 8.0 0.0 46.0 8.0 8.0 4.0 0.0 8.0 8.0 1128.0 367.0 13.0 13.0 0.0 61.0 13.0 13.0 0.0 0.0 8.4 6.4
In [45]:
pm.tail()
Out[45]:
ANALOG PMS_5003 PMS_7003 PMS_A003 SDS_021
AMPHENOL-OUT1 AMPHENOL-OUT2 PPD42-OUT1 PPD42-OUT2 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM10 PM25
2018-02-08 11:35:45+01:00 0.99453 1.0 1.0 0.99863 23.0 23.0 3111.0 887.0 42.0 42.0 0.0 167.0 39.0 39.0 23.0 3.0 22.0 24.0 4014.0 1210.0 41.0 41.0 0.0 238.0 35.0 39.0 25.0 0.0 27.0 32.0 4098.0 1298.0 64.0 73.0 2.0 304.0 49.0 60.0 49.0 13.0 30.7 28.4
2018-02-08 11:35:46+01:00 0.99453 1.0 1.0 0.99863 23.0 23.0 3111.0 887.0 42.0 42.0 0.0 167.0 39.0 39.0 23.0 3.0 22.0 24.0 4014.0 1210.0 41.0 41.0 0.0 238.0 35.0 39.0 25.0 0.0 27.0 32.0 4065.0 1296.0 60.0 67.0 2.0 288.0 48.0 59.0 36.0 8.0 30.7 28.4
2018-02-08 11:35:47+01:00 0.99453 1.0 1.0 0.99863 23.0 23.0 3126.0 882.0 45.0 45.0 1.0 162.0 39.0 39.0 26.0 6.0 23.0 25.0 4035.0 1214.0 40.0 40.0 0.0 238.0 35.0 39.0 25.0 0.0 27.0 32.0 4071.0 1301.0 61.0 68.0 2.0 293.0 47.0 57.0 31.0 10.0 30.6 28.4
2018-02-08 11:35:48+01:00 0.99453 1.0 1.0 0.99863 23.0 23.0 3126.0 882.0 45.0 45.0 1.0 162.0 39.0 39.0 26.0 6.0 23.0 25.0 4035.0 1214.0 40.0 40.0 0.0 238.0 35.0 39.0 25.0 0.0 27.0 32.0 3939.0 1256.0 59.0 65.0 0.0 287.0 47.0 56.0 31.0 8.0 30.6 28.4
2018-02-08 11:35:49+01:00 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 23.0 25.0 4056.0 1229.0 41.0 41.0 0.0 249.0 36.0 40.0 27.0 0.0 27.0 31.0 3729.0 1202.0 57.0 60.0 0.0 264.0 44.0 52.0 34.0 6.0 30.8 28.3

Generowanie agregcji i weryfikacja na wykresie

In [46]:
pmAgg1m =  pm.resample('1min',  label='right', closed='right').mean()
pmAgg10m = pm.resample('10min', label='right', closed='right').mean()
pmAgg1h =  pm.resample('1H',    label='right', closed='right').mean()
In [47]:
plt.plot(pm.loc[:,('PMS_A003', 'PM10A')])
plt.plot(pmAgg10m.loc[:,('PMS_A003', 'PM10A')])
plt.plot(pmAgg1h.loc[:,('PMS_A003', 'PM10A')])
plt.title('PMS_A003: PM10A: \n sample 1s   -   agg 10min   -   agg 1h')
plt.legend()
_=plt.ylabel('ug/m3')
_=plt.xlabel('time')

Zestawinie PM10 10min

In [48]:
plt.plot(pmAgg10m.loc[:,('PMS_5003', 'PM10A')])
plt.plot(pmAgg10m.loc[:,('PMS_7003', 'PM10A')])
plt.plot(pmAgg10m.loc[:,('PMS_A003', 'PM10A')])
plt.plot(pmAgg10m.loc[:,('SDS_021',  'PM10')])
plt.title('PM10')
plt.legend()
_=plt.ylabel('ug/m3')
_=plt.xlabel('time')

Analogi 10min

In [49]:
fig, ax1 = plt.subplots()
ax1.plot(pmAgg10m.loc[:,('PMS_A003', 'PM10A')], 'r')
_=plt.ylabel('ug/m3')
_=plt.xlabel('time')
ax2 = ax1.twinx()
amph1rec = 1.0 / pmAgg10m.loc[:,('ANALOG', 'AMPHENOL-OUT1')]
amph2rec = 1.0 / pmAgg10m.loc[:,('ANALOG', 'AMPHENOL-OUT2')]
shin1rec = 1.0 / pmAgg10m.loc[:,('ANALOG', 'PPD42-OUT1')]
shin2rec = 1.0 / pmAgg10m.loc[:,('ANALOG', 'PPD42-OUT2')]
ax2.plot(amph1rec)
ax2.plot(amph2rec)
ax2.plot(shin1rec)
ax2.plot(shin2rec)
fig.legend()
_ = plt.title('PMS_A003: PM10A    vs.    1.0/ ANALOG')
plt.show(fig)

Odniesienie do pomiarów automatycznych z AM8

In [50]:
am8_10min = pd.read_csv('excento/cas-am8/CAS_export_08.02.2018_14_10min.csv', sep='\t', parse_dates=True, index_col=0)
am8_1h =    pd.read_csv('excento/cas-am8/CAS_export_08.02.2018_14_1h.csv', sep='\t', parse_dates=True, index_col=0)
am8_10min = am8_10min.loc['2018-02-06' : '2018-02-09']
am8_1h =    am8_1h.loc['2018-02-06' : '2018-02-09']
am8_10min = am8_10min.tz_localize('Europe/Warsaw')
am8_1h =    am8_1h.tz_localize('Europe/Warsaw')
In [51]:
plt.plot(pmAgg10m.loc[:,('PMS_A003', 'PM10A')])
plt.plot(am8_10min.loc[:,'008PM10'])
plt.title('PM10 AM8 10min')
plt.legend()
plt.gcf().autofmt_xdate()
_=plt.ylabel('ug/m3')
_=plt.xlabel('time')
In [52]:
plt.plot(pmAgg1h.loc[:,('PMS_A003', 'PM10A')])
plt.plot(am8_1h.loc[:,'008PM10'])
plt.title('PM10 AM8 1h')
plt.legend()
_=plt.ylabel('ug/m3')
_=plt.xlabel('time')

Eksport Excel

In [53]:
# out_samples = pm.loc['2018-02-07'].tz_localize(None)
# out_1m = pmAgg1m.loc['2018-02-07'].tz_localize(None)
# out_10m = pmAgg10m.loc['2018-02-07'].tz_localize(None)
# out_1h = pmAgg1h.loc['2018-02-07'].tz_localize(None)
# writer = pd.ExcelWriter('agg.xlsx')
# out_1m.to_excel(writer,'1min')
# out_10m.to_excel(writer,'10min')
# out_1h.to_excel(writer,'1h')
# writer.save()

# dt = pd.Timedelta('5 days')
# out_dt_samples = out_samples.set_index(out_samples.index - dt)
# out_dt_1m =  out_1m.set_index(out_1m.index - dt)
# out_dt_10m = out_10m.set_index(out_10m.index - dt)
# out_dt_1h =  out_1h.set_index(out_1h.index - dt)
# writer = pd.ExcelWriter('agg_dt.xlsx')
# out_dt_1m.to_excel(writer,'1min')
# out_dt_10m.to_excel(writer,'10min')
# out_dt_1h.to_excel(writer,'1h')
# writer.save()

Analiza statystyczna, regresja liniowa

In [54]:
am8_10min.columns = pd.MultiIndex.from_tuples([('AM8', 'PM10'), ('AM8', 'PM25')])
merged = pd.concat([pmAgg10m.tz_localize(None), am8_10min.tz_localize(None)], axis=1, join='inner')
merged = merged.drop(['ANALOG'], level=0, axis=1).dropna()
merged.head()
Out[54]:
PMS_5003 PMS_7003 PMS_A003 SDS_021 AM8
PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM01A PM01S PM0o3 PM0o5 PM10A PM10S PM10o PM1o0 PM25A PM25S PM2o5 PM5o0 PM10 PM25 PM10 PM25
2018-02-06 14:10:00 10.308594 10.308594 1316.355469 375.197266 18.494141 18.494141 0.523438 63.796875 15.417969 15.417969 6.820312 2.464844 9.078125 9.078125 1597.517578 485.189453 17.015625 17.015625 0.406250 90.419922 15.185547 15.185547 10.011719 2.050781 10.753906 10.753906 1441.417969 457.884766 22.515625 22.515625 1.445312 99.681641 19.666016 19.691406 10.603516 2.705078 10.296094 9.562109 9.33100 3.77333
2018-02-06 14:20:00 11.511719 11.511719 1441.353516 415.832031 20.388672 20.388672 0.511719 68.921875 16.814453 16.814453 7.000000 3.146484 10.488281 10.488281 1799.296875 541.433594 19.035156 19.035156 0.765625 101.498047 16.791016 16.791016 11.160156 3.023438 11.261719 11.261719 1515.644531 478.873047 22.923828 22.923828 1.814453 97.951172 19.953125 19.953125 9.556641 2.822266 10.984766 9.867578 4.70800 5.04667
2018-02-06 14:30:00 11.242188 11.242188 1464.140625 416.691406 20.035156 20.035156 0.785156 70.433594 16.748047 16.748047 7.853516 2.642578 10.585938 10.585938 1834.230469 554.146484 19.880859 19.880859 0.765625 104.658203 17.337891 17.337891 12.205078 3.242188 12.794922 12.794922 1701.310547 539.779297 25.812500 25.812500 1.447266 113.765625 22.863281 22.871094 10.822266 3.095703 12.991602 11.143359 2.32400 2.89333
2018-02-06 14:40:00 12.117417 12.117417 1561.978474 442.802348 19.886497 19.886497 0.493151 65.217221 17.209393 17.209393 6.506849 2.387476 9.733855 9.733855 1760.706458 530.127202 18.839530 18.839530 0.598826 102.285714 16.176125 16.176125 11.878669 3.467710 12.386719 12.386719 1658.185547 525.255859 24.414062 24.414062 1.730469 104.031250 21.298828 21.300781 10.162109 3.115234 12.414481 10.765166 2.25333 3.54833
2018-02-06 14:50:00 11.148438 11.148438 1465.341797 417.783203 19.378906 19.378906 0.751953 63.015625 16.308594 16.308594 6.867188 2.798828 9.826172 9.826172 1751.478516 527.427734 17.392578 17.392578 0.292969 97.960938 15.812500 15.812500 10.023438 1.917969 11.741683 11.741683 1583.941292 501.538160 22.457926 22.457926 1.234834 97.673190 19.943249 19.943249 7.964775 2.632094 11.857227 10.195703 3.94367 4.84667

Macierz korelacji każdy-każdy

In [55]:
_ = sns.heatmap(merged.corr(), cmap="YlGnBu")
_ = plt.title('pairwise correlation of columns method=pearson')
plt.savefig('corr.svg')

Wsp. korelacji względem AM8-PM10

In [56]:
merged.corr().loc[('AM8', 'PM10'), :].sort_values(ascending=False)
Out[56]:
AM8       PM10     1.000000
PMS_5003  PM0o3    0.955509
          PM0o5    0.955123
          PM1o0    0.954641
PMS_A003  PM0o3    0.954560
          PM0o5    0.954529
PMS_5003  PM01S    0.949638
          PM25S    0.949101
          PM2o5    0.948032
PMS_A003  PM1o0    0.947624
          PM01S    0.947589
PMS_5003  PM10S    0.946159
          PM01A    0.945225
          PM25A    0.944159
          PM10A    0.943970
          PM5o0    0.943624
PMS_7003  PM0o3    0.942878
          PM0o5    0.942557
          PM1o0    0.941928
          PM01S    0.939447
PMS_A003  PM01A    0.938825
PMS_7003  PM25S    0.936180
PMS_A003  PM25S    0.935712
PMS_5003  PM10o    0.930097
SDS_021   PM25     0.929522
PMS_7003  PM10S    0.929097
PMS_A003  PM25A    0.927706
PMS_7003  PM01A    0.927333
          PM25A    0.922962
          PM2o5    0.921578
AM8       PM25     0.921503
PMS_A003  PM10S    0.921358
PMS_7003  PM10A    0.918888
PMS_A003  PM10A    0.911970
SDS_021   PM10     0.906646
PMS_A003  PM2o5    0.896827
PMS_7003  PM5o0    0.896598
          PM10o    0.869593
PMS_A003  PM5o0    0.868126
          PM10o    0.861802
Name: (AM8, PM10), dtype: float64

Macierz korelacji dla wszystkich PM10

In [57]:
idx = pd.IndexSlice
pm10 = merged.loc[:, idx[:, ['PM10A', 'PM10']]]
_ = sns.heatmap(pm10.corr(), cmap="YlGnBu")
_ = plt.title('pairwise correlation for PM10 method=pearson')
In [58]:
g = sns.pairplot(pm10, kind="reg", plot_kws=dict(scatter_kws=dict(s=15, alpha=0.6), line_kws=dict(linewidth=2, color='red')))
_ = g.fig.suptitle('pairwise relationships for PM10 with fitted linear regression models')