Data preparation

Functions required to prepare X (and y) from a pandas dataframe.

apply_sliding_window

 apply_sliding_window (data, window_len:Union[int,list],
                       horizon:Union[int,list]=0,
                       x_vars:Union[int,list]=None,
                       y_vars:Union[int,list]=None)

Applies a sliding window on an array-like input to generate a 3d X (and optionally y)

	Type	Default	Details
data			and array-like object with the input data
window_len	int \| list		sliding window length. When using a list, use negative numbers and 0.
horizon	int \| list	0	horizon
x_vars	int \| list	None	indices of the independent variables
y_vars	int \| list	None	indices of the dependent variables (target). [] means no y will be created. None means all variables.

source

prepare_sel_vars_and_steps

 prepare_sel_vars_and_steps (sel_vars=None, sel_steps=None, idxs=False)

source

prepare_idxs

 prepare_idxs (o, shape=None)

data = np.arange(20).reshape(-1,1).repeat(3, 1) * np.array([1, 10, 100])
df = pd.DataFrame(data, columns=['feat_1', 'feat_2', 'feat_3'])
df.head()

	feat_1	feat_2	feat_3
0	0	0	0
1	1	10	100
2	2	20	200
3	3	30	300
4	4	40	400

window_len = 8
horizon = 1
x_vars = None
y_vars = None
X, y = apply_sliding_window(data, window_len, horizon=horizon, x_vars=x_vars, y_vars=y_vars)
print(np.shares_memory(X, data))
print(np.shares_memory(y, data))
print(X.shape, y.shape)
test_eq(X.shape, (len(df) - (window_len - 1 + horizon), df.shape[1], window_len))
test_eq(y.shape, (len(df) - (window_len - 1 + horizon), df.shape[1]))
X[0], y[0]

True
True
(12, 3, 8) (12, 3)

(array([[  0,   1,   2,   3,   4,   5,   6,   7],
        [  0,  10,  20,  30,  40,  50,  60,  70],
        [  0, 100, 200, 300, 400, 500, 600, 700]]),
 array([  8,  80, 800]))

window_len = 8
horizon = 1
x_vars = None
y_vars = 0
X, y = apply_sliding_window(df, window_len, horizon=horizon, x_vars=x_vars, y_vars=y_vars)
print(np.shares_memory(X, df))
print(np.shares_memory(y, df))
print(X.shape, y.shape)
test_eq(X.shape, (len(df) - (window_len - 1 + horizon), df.shape[1], window_len))
test_eq(y.shape, (len(df) - (window_len - 1 + horizon),))
X[0], y[0]

True
True
(12, 3, 8) (12,)

(array([[  0,   1,   2,   3,   4,   5,   6,   7],
        [  0,  10,  20,  30,  40,  50,  60,  70],
        [  0, 100, 200, 300, 400, 500, 600, 700]]),
 8)

window_len = 8
horizon = [1, 2]
x_vars = 0
y_vars = [1, 2]
X, y = apply_sliding_window(df, window_len, horizon=horizon, x_vars=x_vars, y_vars=y_vars)
print(np.shares_memory(X, df))
print(np.shares_memory(y, df))
print(X.shape, y.shape)
test_eq(X.shape, (len(df) - (window_len - 1 + max(horizon)), 1, window_len))
test_eq(y.shape, (len(df) - (window_len - 1 + max(horizon)), len(y_vars), len(horizon)))
X[0], y[0]

True
False
(11, 1, 8) (11, 2, 2)

(array([[0, 1, 2, 3, 4, 5, 6, 7]]),
 array([[ 80,  90],
        [800, 900]]))

window_len = [-4, -2, -1, 0]
horizon = [1, 2, 4]
x_vars = 0
y_vars = [1, 2]
X, y = apply_sliding_window(df, window_len, horizon=horizon, x_vars=x_vars, y_vars=y_vars)
print(np.shares_memory(X, df))
print(np.shares_memory(y, df))
print(X.shape, y.shape)
test_eq(X.shape, (12, 1, 4))
test_eq(y.shape, (12, 2, 3))
X[0], y[0]

False
False
(12, 1, 4) (12, 2, 3)

(array([[0, 2, 3, 4]]),
 array([[ 50,  60,  80],
        [500, 600, 800]]))

source

df2Xy

 df2Xy (df, sample_col=None, feat_col=None, data_cols=None,
        target_col=None, steps_in_rows=False, to3d=True, splits=None,
        sort_by=None, ascending=True, y_func=None, return_names=False)

This function allows you to transform a pandas dataframe into X and y numpy arrays that can be used to create a TSDataset. sample_col: column that uniquely identifies each sample. feat_col: used for multivariate datasets. It indicates which is the column that indicates the feature by row. data_col: indicates ths column/s where the data is located. If None, it means all columns (except the sample_col, feat_col, and target_col) target_col: indicates the column/s where the target is. steps_in_rows: flag to indicate if each step is in a different row or in a different column (default). to3d: turns X to 3d (including univariate time series) sort_by: this is used to pass any colum/s that are needed to sort the steps in the sequence. If you pass a sample_col and/ or feat_col these will be automatically used before the sort_by column/s, and you don’t need to add them to the sort_by column/s list. y_func: function used to calculate y for each sample (and target_col) return_names: flag to return the names of the columns from where X was generated

source

split_Xy

 split_Xy (X, y=None, splits=None)

df = pd.DataFrame()
df['sample_id'] = np.array([1,1,1,2,2,2,3,3,3])
df['var1'] = df['sample_id'] * 10 + df.index.values
df['var2'] = df['sample_id'] * 100 + df.index.values
df

	sample_id	var1	var2
0	1	10	100
1	1	11	101
2	1	12	102
3	2	23	203
4	2	24	204
5	2	25	205
6	3	36	306
7	3	37	307
8	3	38	308

X_df, y_df = df2Xy(df, sample_col='sample_id', steps_in_rows=True)
test_eq(X_df[0], np.array([[10, 11, 12], [100, 101, 102]]))

n_samples = 1_000
n_rows = 10_000

sample_ids = np.arange(n_samples).repeat(n_rows//n_samples).reshape(-1,1)
feat_ids = np.tile(np.arange(n_rows // n_samples), n_samples).reshape(-1,1)
cont = np.random.randn(n_rows, 6)
ind_cat = np.random.randint(0, 3, (n_rows, 1))
target = np.array([0,1,2])[ind_cat]
ind_cat2 = np.random.randint(0, 3, (n_rows, 1))
target2 = np.array([100,200,300])[ind_cat2]
data = np.concatenate([sample_ids, feat_ids, cont, target, target], -1)
columns = ['sample_id', 'feat_id'] + (np.arange(6) + 1).astype(str).tolist() + ['target'] + ['target2']
df = pd.DataFrame(data, columns=columns)
idx = random_choice(np.arange(len(df)), len(df), False)
new_dtypes = {'sample_id':np.int32, 'feat_id':np.int32, '1':np.float32, '2':np.float32, '3':np.float32, '4':np.float32, '5':np.float32, '6':np.float32}
df = df.astype(dtype=new_dtypes)
df = df.loc[idx].reset_index(drop=True)
df

	sample_id	feat_id	1	2	3	4	5	6	target	target2
0	625	2	-1.390549	0.770179	-0.848480	0.853631	-0.309984	0.874338	2.0	2.0
1	526	4	1.152397	2.064397	-0.392603	-0.275797	-0.047526	-2.248814	2.0	2.0
2	397	6	-1.052930	0.631396	-0.758800	-0.606483	-2.776054	-0.457755	1.0	1.0
3	528	8	-0.178637	-1.253319	-1.154014	0.913876	1.051010	-0.635762	1.0	1.0
4	249	2	0.612595	0.888297	0.065024	1.621935	-0.180479	0.309977	1.0	1.0
...	...	...	...	...	...	...	...	...	...	...
9995	272	1	-0.432325	1.645262	1.502872	-1.144859	0.919653	0.414304	0.0	0.0
9996	920	5	-0.724702	-1.471832	1.209086	1.206532	0.555676	0.352726	2.0	2.0
9997	662	6	1.122043	-0.379357	-0.344517	-1.545091	0.187894	1.062510	2.0	2.0
9998	71	7	-0.053582	-0.854992	-1.118632	-1.967820	-0.344804	0.128105	0.0	0.0
9999	407	4	-1.565716	-0.947183	-0.401944	-1.309024	-0.237755	-0.743251	2.0	2.0

10000 rows × 10 columns

from scipy.stats import mode

def y_func(o): return mode(o, axis=1, keepdims=True).mode
X, y = df2xy(df, sample_col='sample_id', feat_col='feat_id', target_col=['target', 'target2'], sort_by=['sample_id', 'feat_id'], y_func=y_func)
test_eq(X.shape, (1000, 10, 6))
test_eq(y.shape, (1000, 2))
rand_idx = np.random.randint(0, np.max(df.sample_id))
sorted_df = df.sort_values(by=['sample_id', 'feat_id'], kind='stable').reset_index(drop=True)
test_eq(X[rand_idx], sorted_df[sorted_df.sample_id == rand_idx][['1', '2', '3', '4', '5', '6']].values)
test_eq(np.squeeze(mode(sorted_df[sorted_df.sample_id == rand_idx][['target', 'target2']].values).mode), y[rand_idx])

# Univariate
from io import StringIO

TESTDATA = StringIO("""sample_id;value_0;value_1;target
    rob;2;3;0
    alice;6;7;1
    eve;11;12;2
    """)

df = pd.read_csv(TESTDATA, sep=";")
display(df)
X, y = df2Xy(df, sample_col='sample_id', target_col='target', data_cols=['value_0', 'value_1'], sort_by='sample_id')
test_eq(X.shape, (3, 1, 2))
test_eq(y.shape, (3,))
X, y

	sample_id	value_0	value_1	target
0	rob	2	3	0
1	alice	6	7	1
2	eve	11	12	2

(array([[[ 6,  7]],
 
        [[11, 12]],
 
        [[ 2,  3]]]),
 array([1, 2, 0]))

# Univariate
TESTDATA = StringIO("""sample_id;timestep;values;target
    rob;1;2;0
    alice;1;6;1
    eve;1;11;2
    
    rob;2;3;0
    alice;2;7;1
    eve;2;12;2
    """)

df = pd.read_csv(TESTDATA, sep=";")
display(df)
def y_func(o): return mode(o, axis=1).mode
X, y = df2xy(df, sample_col='sample_id', target_col='target', data_cols=['values'], sort_by='timestep', to3d=True, y_func=y_func)
test_eq(X.shape, (3, 1, 2))
test_eq(y.shape, (3, ))
print(X, y)

	sample_id	timestep	values	target
0	rob	1	2	0
1	alice	1	6	1
2	eve	1	11	2
3	rob	2	3	0
4	alice	2	7	1
5	eve	2	12	2

[[[ 6  7]]

 [[11 12]]

 [[ 2  3]]] [1 2 0]

# Multivariate
TESTDATA = StringIO("""sample_id;trait;value_0;value_1;target
    rob;green;2;3;0
    rob;yellow;3;4;0
    rob;blue;4;5;0
    rob;red;5;6;0
    alice;green;6;7;1
    alice;yellow;7;8;1
    alice;blue;8;9;1
    alice;red;9;10;1
    eve;yellow;11;12;2
    eve;green;10;11;2
    eve;blue;12;12;2
    eve;red;13;14;2
    """)

df = pd.read_csv(TESTDATA, sep=";")
idx = random_choice(len(df), len(df), False)
df = df.iloc[idx]
display(df)
def y_func(o): return mode(o, axis=1).mode
X, y = df2xy(df, sample_col='sample_id', feat_col='trait', target_col='target', data_cols=['value_0', 'value_1'], y_func=y_func)
print(X, y)
test_eq(X.shape, (3, 4, 2))
test_eq(y.shape, (3,))

	sample_id	trait	value_0	value_1	target
9	eve	green	10	11	2
10	eve	blue	12	12	2
3	rob	red	5	6	0
0	rob	green	2	3	0
6	alice	blue	8	9	1
2	rob	blue	4	5	0
1	rob	yellow	3	4	0
4	alice	green	6	7	1
7	alice	red	9	10	1
8	eve	yellow	11	12	2
11	eve	red	13	14	2
5	alice	yellow	7	8	1

[[[ 8  9]
  [ 6  7]
  [ 9 10]
  [ 7  8]]

 [[12 12]
  [10 11]
  [13 14]
  [11 12]]

 [[ 4  5]
  [ 2  3]
  [ 5  6]
  [ 3  4]]] [1 2 0]

# Multivariate, multi-label
TESTDATA = StringIO("""sample_id;trait;value_0;value_1;target1;target2
    rob;green;2;3;0;0
    rob;yellow;3;4;0;0
    rob;blue;4;5;0;0
    rob;red;5;6;0;0
    alice;green;6;7;1;0
    alice;yellow;7;8;1;0
    alice;blue;8;9;1;0
    alice;red;9;10;1;0
    eve;yellow;11;12;2;1
    eve;green;10;11;2;1
    eve;blue;12;12;2;1
    eve;red;13;14;2;1
    """)

df = pd.read_csv(TESTDATA, sep=";")
display(df)
def y_func(o): return mode(o, axis=1, keepdims=True).mode
X, y = df2xy(df, sample_col='sample_id', feat_col='trait', target_col=['target1', 'target2'], data_cols=['value_0', 'value_1'], y_func=y_func)
test_eq(X.shape, (3, 4, 2))
test_eq(y.shape, (3, 2))
print(X, y)

	sample_id	trait	value_0	value_1	target1	target2
0	rob	green	2	3	0	0
1	rob	yellow	3	4	0	0
2	rob	blue	4	5	0	0
3	rob	red	5	6	0	0
4	alice	green	6	7	1	0
5	alice	yellow	7	8	1	0
6	alice	blue	8	9	1	0
7	alice	red	9	10	1	0
8	eve	yellow	11	12	2	1
9	eve	green	10	11	2	1
10	eve	blue	12	12	2	1
11	eve	red	13	14	2	1

[[[ 8  9]
  [ 6  7]
  [ 9 10]
  [ 7  8]]

 [[12 12]
  [10 11]
  [13 14]
  [11 12]]

 [[ 4  5]
  [ 2  3]
  [ 5  6]
  [ 3  4]]] [[1 0]
 [2 1]
 [0 0]]

# Multivariate, unlabeled
TESTDATA = StringIO("""sample_id;trait;value_0;value_1;target
    rob;green;2;3;0
    rob;yellow;3;4;0
    rob;blue;4;5;0
    rob;red;5;6;0
    alice;green;6;7;1
    alice;yellow;7;8;1
    alice;blue;8;9;1
    alice;red;9;10;1
    eve;yellow;11;12;2
    eve;green;10;11;2
    eve;blue;12;12;2
    eve;red;13;14;2
    """)

df = pd.read_csv(TESTDATA, sep=";")
idx = random_choice(len(df), len(df), False)
df = df.iloc[idx]
display(df)
def y_func(o): return mode(o, axis=1, keepdims=True).mode
X, y = df2xy(df, sample_col='sample_id', feat_col='trait', data_cols=['value_0', 'value_1'], y_func=y_func)
print(X, y)
test_eq(X.shape, (3, 4, 2))
test_eq(y, None)

	sample_id	trait	value_0	value_1	target
11	eve	red	13	14	2
3	rob	red	5	6	0
9	eve	green	10	11	2
10	eve	blue	12	12	2
6	alice	blue	8	9	1
1	rob	yellow	3	4	0
4	alice	green	6	7	1
2	rob	blue	4	5	0
0	rob	green	2	3	0
8	eve	yellow	11	12	2
7	alice	red	9	10	1
5	alice	yellow	7	8	1

[[[ 8  9]
  [ 6  7]
  [ 9 10]
  [ 7  8]]

 [[12 12]
  [10 11]
  [13 14]
  [11 12]]

 [[ 4  5]
  [ 2  3]
  [ 5  6]
  [ 3  4]]] None

TESTDATA = StringIO("""sample_id;trait;timestep;values;target
    rob;green;1;2;0
    rob;yellow;1;3;0
    rob;blue;1;4;0
    rob;red;1;5;0
    alice;green;1;6;1
    alice;yellow;1;7;1
    alice;blue;1;8;1
    alice;red;1;9;1
    eve;yellow;1;11;2
    eve;green;1;10;2
    eve;blue;1;12;2
    eve;red;1;13;2
    
    rob;green;2;3;0
    rob;yellow;2;4;0
    rob;blue;2;5;0
    rob;red;2;6;0
    alice;green;2;7;1
    alice;yellow;2;8;1
    alice;blue;2;9;1
    alice;red;2;10;1
    eve;yellow;2;12;2
    eve;green;2;11;2
    eve;blue;2;13;2
    eve;red;2;14;2
    """)

df = pd.read_csv(TESTDATA, sep=";")
display(df)
def y_func(o): return mode(o, axis=1).mode
X, y = df2xy(df, sample_col='sample_id', feat_col='trait', sort_by='timestep', target_col='target', data_cols=['values'], y_func=y_func)
print(X, y)
test_eq(X.shape, (3, 4, 2))
test_eq(y.shape, (3, ))

	sample_id	trait	timestep	values	target
0	rob	green	1	2	0
1	rob	yellow	1	3	0
2	rob	blue	1	4	0
3	rob	red	1	5	0
4	alice	green	1	6	1
5	alice	yellow	1	7	1
6	alice	blue	1	8	1
7	alice	red	1	9	1
8	eve	yellow	1	11	2
9	eve	green	1	10	2
10	eve	blue	1	12	2
11	eve	red	1	13	2
12	rob	green	2	3	0
13	rob	yellow	2	4	0
14	rob	blue	2	5	0
15	rob	red	2	6	0
16	alice	green	2	7	1
17	alice	yellow	2	8	1
18	alice	blue	2	9	1
19	alice	red	2	10	1
20	eve	yellow	2	12	2
21	eve	green	2	11	2
22	eve	blue	2	13	2
23	eve	red	2	14	2

[[[ 8  9]
  [ 6  7]
  [ 9 10]
  [ 7  8]]

 [[12 13]
  [10 11]
  [13 14]
  [11 12]]

 [[ 4  5]
  [ 2  3]
  [ 5  6]
  [ 3  4]]] [1 2 0]

source

df2np3d

 df2np3d (df, groupby, data_cols=None)

Transforms a df (with the same number of rows per group in groupby) to a 3d ndarray

user = np.array([1,2]).repeat(4).reshape(-1,1)
val = np.random.rand(8, 3)
data = np.concatenate([user, val], axis=-1)
df = pd.DataFrame(data, columns=['user', 'x1', 'x2', 'x3'])
test_eq(df2np3d(df, ['user'], ['x1', 'x2', 'x3']).shape, (2, 3, 4))

source

add_missing_value_cols

 add_missing_value_cols (df, cols=None, dtype=<class 'float'>,
                         fill_value=None)

data = np.random.randn(10, 2)
mask = data > .8
data[mask] = np.nan
df = pd.DataFrame(data, columns=['A', 'B'])
df = add_missing_value_cols(df, cols=None, dtype=float)
test_eq(df['A'].isnull().sum(), df['missing_A'].sum())
test_eq(df['B'].isnull().sum(), df['missing_B'].sum())
df

	A	B	missing_A	missing_B
0	0.476712	-0.880797	0.0	0.0
1	NaN	-1.517210	1.0	0.0
2	-1.348997	-0.878441	0.0	0.0
3	NaN	0.290756	1.0	0.0
4	0.569218	-1.415777	0.0	0.0
5	0.591641	-2.133860	0.0	0.0
6	NaN	NaN	1.0	1.0
7	NaN	-0.119397	1.0	0.0
8	-0.727988	0.057254	0.0	0.0
9	-0.631352	-0.219028	0.0	0.0

source

add_missing_timestamps

 add_missing_timestamps (df, datetime_col=None, use_index=False,
                         unique_id_cols=None, groupby=None,
                         fill_value=nan, range_by_group=True,
                         start_date=None, end_date=None, freq=None)

	Type	Default	Details
df			pandas DataFrame
datetime_col	NoneType	None	column that contains the datetime data (without duplicates within groups)
use_index	bool	False	indicates if the index contains the datetime data
unique_id_cols	NoneType	None	column used to identify unique_ids
groupby	NoneType	None	same as unique_id_cols. Will be deprecated. Kept for compatiblity.
fill_value	float	nan	values that will be insert where missing dates exist. Default:np.nan
range_by_group	bool	True	if True, dates will be filled between min and max dates for each group. Otherwise, between the min and max dates in the df.
start_date	NoneType	None	start date to fill in missing dates (same for all unique_ids)
end_date	NoneType	None	end date to fill in missing dates (same for all unique_ids)
freq	NoneType	None	frequency used to fill in the missing datetime

# Filling dates between min and max dates
dates = pd.date_range('2021-05-01', '2021-05-07').values
data = np.zeros((len(dates), 3))
data[:, 0] = dates
data[:, 1] = np.random.rand(len(dates))
data[:, 2] = np.random.rand(len(dates))
cols = ['date', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([1,3]).reset_index(drop=True)
date_df_with_missing_dates

	date	feature1	feature2
0	2021-05-01	0.537248	0.670897
1	2021-05-03	0.299912	0.421039
2	2021-05-05	0.648372	0.204641
3	2021-05-06	0.017475	0.022183
4	2021-05-07	0.965919	0.470055

# No groups
expected_output_df = date_df.copy()
expected_output_df.loc[[1,3], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(), 
                                   'date', 
                                   unique_id_cols=None, 
                                   fill_value=np.nan, 
                                   range_by_group=False)
test_eq(output_df, expected_output_df)

	date	feature1	feature2
0	2021-05-01	0.537248	0.670897
1	2021-05-02	NaN	NaN
2	2021-05-03	0.299912	0.421039
3	2021-05-04	NaN	NaN
4	2021-05-05	0.648372	0.204641
5	2021-05-06	0.017475	0.022183
6	2021-05-07	0.965919	0.470055

# Filling dates between min and max dates for each value in groupby column
dates = pd.date_range('2021-05-01', '2021-05-07').values
dates = np.concatenate((dates, dates))
data = np.zeros((len(dates), 4))
data[:, 0] = dates
data[:, 1] = np.array([0]*(len(dates)//2)+[1]*(len(dates)//2))
data[:, 2] = np.random.rand(len(dates))
data[:, 3] = np.random.rand(len(dates))
cols = ['date', 'id', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'id': int, 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([0,1,3,8,11,13]).reset_index(drop=True)
date_df_with_missing_dates

	date	id	feature1	feature2
0	2021-05-03	0	0.059398	0.255853
1	2021-05-05	0	0.235536	0.455261
2	2021-05-06	0	0.724423	0.280910
3	2021-05-07	0	0.303682	0.853959
4	2021-05-01	1	0.022424	0.408510
5	2021-05-03	1	0.508190	0.603880
6	2021-05-04	1	0.330924	0.108156
7	2021-05-06	1	0.601481	0.020182

# groupby='id', range_by_group=True
expected_output_df = date_df.drop([0,1,13]).reset_index(drop=True)  
expected_output_df.loc[[1,6,9], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(), 
                                   'date', 
                                   unique_id_cols='id', 
                                   fill_value=np.nan, 
                                   range_by_group=True)
test_eq(expected_output_df, output_df)

	date	id	feature1	feature2
0	2021-05-03	0	0.059398	0.255853
1	2021-05-04	0	NaN	NaN
2	2021-05-05	0	0.235536	0.455261
3	2021-05-06	0	0.724423	0.280910
4	2021-05-07	0	0.303682	0.853959
5	2021-05-01	1	0.022424	0.408510
6	2021-05-02	1	NaN	NaN
7	2021-05-03	1	0.508190	0.603880
8	2021-05-04	1	0.330924	0.108156
9	2021-05-05	1	NaN	NaN
10	2021-05-06	1	0.601481	0.020182

# groupby='id', range_by_group=False
expected_output_df = date_df.copy() 
expected_output_df.loc[[0,1,3,8,11,13], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(), 
                                   'date', 
                                   unique_id_cols='id', 
                                   fill_value=np.nan, 
                                   range_by_group=False)
test_eq(expected_output_df, output_df)

	date	id	feature1	feature2
0	2021-05-01	0	NaN	NaN
1	2021-05-02	0	NaN	NaN
2	2021-05-03	0	0.059398	0.255853
3	2021-05-04	0	NaN	NaN
4	2021-05-05	0	0.235536	0.455261
5	2021-05-06	0	0.724423	0.280910
6	2021-05-07	0	0.303682	0.853959
7	2021-05-01	1	0.022424	0.408510
8	2021-05-02	1	NaN	NaN
9	2021-05-03	1	0.508190	0.603880
10	2021-05-04	1	0.330924	0.108156
11	2021-05-05	1	NaN	NaN
12	2021-05-06	1	0.601481	0.020182
13	2021-05-07	1	NaN	NaN

# Filling dates between min and max timestamps
dates = pd.date_range('2021-05-01 000:00', '2021-05-01 20:00', freq='4H').values
data = np.zeros((len(dates), 3))
data[:, 0] = dates
data[:, 1] = np.random.rand(len(dates))
data[:, 2] = np.random.rand(len(dates))
cols = ['date', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([1,3]).reset_index(drop=True)
date_df_with_missing_dates

	date	feature1	feature2
0	2021-05-01 00:00:00	0.774846	0.624488
1	2021-05-01 08:00:00	0.683837	0.441230
2	2021-05-01 16:00:00	0.142269	0.279095
3	2021-05-01 20:00:00	0.953686	0.205123

# No groups
expected_output_df = date_df.copy()
expected_output_df.loc[[1,3], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(), 'date', groupby=None, fill_value=np.nan, range_by_group=False, freq='4H')
test_eq(output_df, expected_output_df)

	date	feature1	feature2
0	2021-05-01 00:00:00	0.774846	0.624488
1	2021-05-01 04:00:00	NaN	NaN
2	2021-05-01 08:00:00	0.683837	0.441230
3	2021-05-01 12:00:00	NaN	NaN
4	2021-05-01 16:00:00	0.142269	0.279095
5	2021-05-01 20:00:00	0.953686	0.205123

# Filling missing values between min and max timestamps for each value in groupby column

dates = pd.date_range('2021-05-01 000:00', '2021-05-01 20:00', freq='4H').values
dates = np.concatenate((dates, dates))
data = np.zeros((len(dates), 4))
data[:, 0] = dates
data[:, 1] = np.array([0]*(len(dates)//2)+[1]*(len(dates)//2))
data[:, 2] = np.random.rand(len(dates))
data[:, 3] = np.random.rand(len(dates))
cols = ['date', 'id', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'id': int, 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([0,1,3,8,9,11]).reset_index(drop=True)
date_df_with_missing_dates

	date	id	feature1	feature2
0	2021-05-01 08:00:00	0	0.438784	0.084472
1	2021-05-01 16:00:00	0	0.059613	0.445215
2	2021-05-01 20:00:00	0	0.511807	0.001034
3	2021-05-01 00:00:00	1	0.970115	0.280121
4	2021-05-01 04:00:00	1	0.775051	0.436359
5	2021-05-01 16:00:00	1	0.469987	0.457442

# groupby='id', range_by_group=True
expected_output_df = date_df.drop([0,1,11]).reset_index(drop=True)  
expected_output_df.loc[[1,6,7], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(),
                                   'date', 
                                   groupby='id', 
                                   fill_value=np.nan, 
                                   range_by_group=True, 
                                   freq='4H')
test_eq(expected_output_df, output_df)

	date	id	feature1	feature2
0	2021-05-01 08:00:00	0	0.438784	0.084472
1	2021-05-01 12:00:00	0	NaN	NaN
2	2021-05-01 16:00:00	0	0.059613	0.445215
3	2021-05-01 20:00:00	0	0.511807	0.001034
4	2021-05-01 00:00:00	1	0.970115	0.280121
5	2021-05-01 04:00:00	1	0.775051	0.436359
6	2021-05-01 08:00:00	1	NaN	NaN
7	2021-05-01 12:00:00	1	NaN	NaN
8	2021-05-01 16:00:00	1	0.469987	0.457442

# groupby='id', range_by_group=False
expected_output_df = date_df.copy() 
expected_output_df.loc[[0,1,3,8,9,11], ['feature1', 'feature2']] = np.nan
display(expected_output_df)
output_df = add_missing_timestamps(date_df_with_missing_dates.copy(), 
                                   'date', 
                                   groupby='id', 
                                   fill_value=np.nan, 
                                   range_by_group=False, 
                                   freq='4H')
test_eq(expected_output_df, output_df)

	date	id	feature1	feature2
0	2021-05-01 00:00:00	0	NaN	NaN
1	2021-05-01 04:00:00	0	NaN	NaN
2	2021-05-01 08:00:00	0	0.438784	0.084472
3	2021-05-01 12:00:00	0	NaN	NaN
4	2021-05-01 16:00:00	0	0.059613	0.445215
5	2021-05-01 20:00:00	0	0.511807	0.001034
6	2021-05-01 00:00:00	1	0.970115	0.280121
7	2021-05-01 04:00:00	1	0.775051	0.436359
8	2021-05-01 08:00:00	1	NaN	NaN
9	2021-05-01 12:00:00	1	NaN	NaN
10	2021-05-01 16:00:00	1	0.469987	0.457442
11	2021-05-01 20:00:00	1	NaN	NaN

# No groups, with duplicate dates ==> FAILS
dates = pd.date_range('2021-05-01 000:00', '2021-05-01 20:00', freq='4H').values
data = np.zeros((len(dates), 3))
data[:, 0] = dates
data[:, 1] = np.random.rand(len(dates))
data[:, 2] = np.random.rand(len(dates))
cols = ['date', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([1,3]).reset_index(drop=True)
date_df_with_missing_dates.loc[3, 'date'] = date_df_with_missing_dates.loc[2, 'date']
display(date_df_with_missing_dates)
test_fail(add_missing_timestamps, args=[date_df_with_missing_dates, 'date'], kwargs=dict(groupby=None, fill_value=np.nan, range_by_group=False, freq='4H'), )

	date	feature1	feature2
0	2021-05-01 00:00:00	0.755092	0.002068
1	2021-05-01 08:00:00	0.570693	0.087019
2	2021-05-01 16:00:00	0.228869	0.856618
3	2021-05-01 16:00:00	0.349506	0.428253

# groupby='id', range_by_group=True, with duplicate dates ==> FAILS

dates = pd.date_range('2021-05-01 000:00', '2021-05-01 20:00', freq='4H').values
dates = np.concatenate((dates, dates))
data = np.zeros((len(dates), 4))
data[:, 0] = dates
data[:, 1] = np.array([0]*(len(dates)//2)+[1]*(len(dates)//2))
data[:, 2] = np.random.rand(len(dates))
data[:, 3] = np.random.rand(len(dates))
cols = ['date', 'id', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'id': int, 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([0,1,8,9,11]).reset_index(drop=True)
date_df_with_missing_dates.loc[3, 'date'] = date_df_with_missing_dates.loc[2, 'date']
display(date_df_with_missing_dates)
test_fail(add_missing_timestamps, args=[date_df_with_missing_dates, 'date'], kwargs=dict(groupby='id', fill_value=np.nan, range_by_group=True, freq='4H'), 
          contains='cannot handle a non-unique multi-index!')

	date	id	feature1	feature2
0	2021-05-01 08:00:00	0	0.040345	0.312874
1	2021-05-01 12:00:00	0	0.713424	0.597211
2	2021-05-01 16:00:00	0	0.468382	0.652314
3	2021-05-01 16:00:00	0	0.396691	0.605664
4	2021-05-01 00:00:00	1	0.804646	0.964115
5	2021-05-01 04:00:00	1	0.089925	0.072410
6	2021-05-01 16:00:00	1	0.830786	0.560658

# groupby='id', range_by_group=FALSE, with duplicate dates ==> FAILS

dates = pd.date_range('2021-05-01 000:00', '2021-05-01 20:00', freq='4H').values
dates = np.concatenate((dates, dates))
data = np.zeros((len(dates), 4))
data[:, 0] = dates
data[:, 1] = np.array([0]*(len(dates)//2)+[1]*(len(dates)//2))
data[:, 2] = np.random.rand(len(dates))
data[:, 3] = np.random.rand(len(dates))
cols = ['date', 'id', 'feature1', 'feature2']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'id': int, 'feature1': float, 'feature2': float})
date_df_with_missing_dates = date_df.drop([0,1,8,9,11]).reset_index(drop=True)
date_df_with_missing_dates.loc[3, 'date'] = date_df_with_missing_dates.loc[2, 'date']
display(date_df_with_missing_dates)
test_fail(add_missing_timestamps, args=[date_df_with_missing_dates, 'date'], kwargs=dict(groupby='id', fill_value=np.nan, range_by_group=False, freq='4H'), 
          contains='cannot handle a non-unique multi-index!')

	date	id	feature1	feature2
0	2021-05-01 08:00:00	0	0.448508	0.953596
1	2021-05-01 12:00:00	0	0.868802	0.526845
2	2021-05-01 16:00:00	0	0.223070	0.304842
3	2021-05-01 16:00:00	0	0.645661	0.270956
4	2021-05-01 00:00:00	1	0.017250	0.787757
5	2021-05-01 04:00:00	1	0.783341	0.608269
6	2021-05-01 16:00:00	1	0.426247	0.926149

source

time_encoding

 time_encoding (series, freq, max_val=None)

Transforms a pandas series of dtype datetime64 (of any freq) or DatetimeIndex into 2 float arrays

Available options: microsecond, millisecond, second, minute, hour, day = day_of_month = dayofmonth, day_of_week = weekday = dayofweek, day_of_year = dayofyear, week = week_of_year = weekofyear, month and year

for freq in ['microsecond', 'second', 'minute', 'hour', 'day', 'dayofweek', 'dayofyear', 'month']:
    tdf = pd.DataFrame(pd.date_range('2021-03-01', dt.datetime.today()), columns=['date'])
    a,b = time_encoding(tdf.date, freq=freq)
    plt.plot(a)
    plt.plot(b)
    plt.title(freq)
    plt.show()

for freq in ['microsecond', 'second', 'minute', 'hour', 'day', 'dayofweek', 'dayofyear', 'month']:
    dateindex = pd.date_range('2021-03-01', dt.datetime.today())
    a,b = time_encoding(dateindex, freq=freq)
    plt.plot(a)
    plt.plot(b)
    plt.title(freq)
    plt.show()

dow_sin, dow_cos = time_encoding(date_df['date'], 'dayofweek')
plt.plot(dow_sin)
plt.plot(dow_cos)
plt.title('DayOfWeek')
plt.show()
date_df['dow_sin'] = dow_sin
date_df['dow_cos'] = dow_cos
date_df

	date	id	feature1	feature2	dow_sin	dow_cos
0	2021-05-01 00:00:00	0	0.773597	0.465634	-0.974928	-0.222521
1	2021-05-01 04:00:00	0	0.265526	0.963753	-0.974928	-0.222521
2	2021-05-01 08:00:00	0	0.448508	0.953596	-0.974928	-0.222521
3	2021-05-01 12:00:00	0	0.868802	0.526845	-0.974928	-0.222521
4	2021-05-01 16:00:00	0	0.223070	0.304842	-0.974928	-0.222521
5	2021-05-01 20:00:00	0	0.645661	0.270956	-0.974928	-0.222521
6	2021-05-01 00:00:00	1	0.017250	0.787757	-0.974928	-0.222521
7	2021-05-01 04:00:00	1	0.783341	0.608269	-0.974928	-0.222521
8	2021-05-01 08:00:00	1	0.629875	0.170726	-0.974928	-0.222521
9	2021-05-01 12:00:00	1	0.302927	0.682136	-0.974928	-0.222521
10	2021-05-01 16:00:00	1	0.426247	0.926149	-0.974928	-0.222521
11	2021-05-01 20:00:00	1	0.830624	0.543715	-0.974928	-0.222521

source

get_gaps

 get_gaps (o:torch.Tensor, forward:bool=True, backward:bool=True,
           nearest:bool=True, normalize:bool=True)

Number of sequence steps from previous, to next and/or to nearest real value along the last dimension of 3D arrays or tensors

source

nearest_gaps

 nearest_gaps (o, normalize=True)

Number of sequence steps to nearest real value along the last dimension of 3D arrays or tensors

source

backward_gaps

 backward_gaps (o, normalize=True)

Number of sequence steps to next real value along the last dimension of 3D arrays or tensors

source

forward_gaps

 forward_gaps (o, normalize=True)

Number of sequence steps since previous real value along the last dimension of 3D arrays or tensors

t = torch.rand(1, 2, 8)
arr = t.numpy()
t[t <.6] = np.nan
test_ge(nearest_gaps(t).min().item(), 0)
test_ge(nearest_gaps(arr).min(), 0)
test_le(nearest_gaps(t).min().item(), 1)
test_le(nearest_gaps(arr).min(), 1)
test_eq(torch.isnan(forward_gaps(t)).sum(), 0)
test_eq(np.isnan(forward_gaps(arr)).sum(), 0)
ag = get_gaps(t)
test_eq(ag.shape, (1,6,8))
test_eq(torch.isnan(ag).sum(), 0)

source

add_delta_timestamp_cols

 add_delta_timestamp_cols (df, cols=None, groupby=None, forward=True,
                           backward=True, nearest=True, normalize=True)

# Add delta timestamp features for the no groups setting
dates = pd.date_range('2021-05-01', '2021-05-07').values
data = np.zeros((len(dates), 2))
data[:, 0] = dates
data[:, 1] = np.random.rand(len(dates))

cols = ['date', 'feature1']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'feature1': float})
date_df.loc[[1,3,4],'feature1'] = np.nan
date_df

	date	feature1
0	2021-05-01	0.132532
1	2021-05-02	NaN
2	2021-05-03	0.403176
3	2021-05-04	NaN
4	2021-05-05	NaN
5	2021-05-06	0.179554
6	2021-05-07	0.446536

# No groups
expected_output_df = date_df.copy()
expected_output_df['feature1_dt_fwd'] = np.array([1,1,2,1,2,3,1])
expected_output_df['feature1_dt_bwd'] = np.array([2,1,3,2,1,1,1])
expected_output_df['feature1_dt_nearest'] = np.array([1,1,2,1,1,1,1])

display(expected_output_df)
output_df = add_delta_timestamp_cols(date_df, cols='feature1', normalize=False)
test_eq(expected_output_df, output_df)

	date	feature1	feature1_dt_fwd	feature1_dt_bwd	feature1_dt_nearest
0	2021-05-01	0.132532	1	2	1
1	2021-05-02	NaN	1	1	1
2	2021-05-03	0.403176	2	3	2
3	2021-05-04	NaN	1	2	1
4	2021-05-05	NaN	2	1	1
5	2021-05-06	0.179554	3	1	1
6	2021-05-07	0.446536	1	1	1

# Add delta timestamp features within a group
dates = pd.date_range('2021-05-01', '2021-05-07').values
dates = np.concatenate((dates, dates))
data = np.zeros((len(dates), 3))
data[:, 0] = dates
data[:, 1] = np.array([0]*(len(dates)//2)+[1]*(len(dates)//2))
data[:, 2] = np.random.rand(len(dates))

cols = ['date', 'id', 'feature1']
date_df = pd.DataFrame(data, columns=cols).astype({'date': 'datetime64[ns]', 'id': int, 'feature1': float})
date_df.loc[[1,3,4,8,9,11],'feature1'] = np.nan
date_df

	date	id	feature1
0	2021-05-01	0	0.405327
1	2021-05-02	0	NaN
2	2021-05-03	0	0.055934
3	2021-05-04	0	NaN
4	2021-05-05	0	NaN
5	2021-05-06	0	0.698408
6	2021-05-07	0	0.064831
7	2021-05-01	1	0.407541
8	2021-05-02	1	NaN
9	2021-05-03	1	NaN
10	2021-05-04	1	0.113590
11	2021-05-05	1	NaN
12	2021-05-06	1	0.548088
13	2021-05-07	1	0.348813

# groupby='id'
expected_output_df = date_df.copy()
expected_output_df['feature1_dt_fwd'] = np.array([1,1,2,1,2,3,1,1,1,2,3,1,2,1])
expected_output_df['feature1_dt_bwd'] = np.array([2,1,3,2,1,1,1,3,2,1,2,1,1,1])
expected_output_df['feature1_dt_nearest'] = np.array([1,1,2,1,1,1,1,1,1,1,2,1,1,1])

display(expected_output_df)
output_df = add_delta_timestamp_cols(date_df, cols='feature1', groupby='id', normalize=False)
test_eq(expected_output_df, output_df)

	date	id	feature1	feature1_dt_fwd	feature1_dt_bwd	feature1_dt_nearest
0	2021-05-01	0	0.405327	1	2	1
1	2021-05-02	0	NaN	1	1	1
2	2021-05-03	0	0.055934	2	3	2
3	2021-05-04	0	NaN	1	2	1
4	2021-05-05	0	NaN	2	1	1
5	2021-05-06	0	0.698408	3	1	1
6	2021-05-07	0	0.064831	1	1	1
7	2021-05-01	1	0.407541	1	3	1
8	2021-05-02	1	NaN	1	2	1
9	2021-05-03	1	NaN	2	1	1
10	2021-05-04	1	0.113590	3	2	2
11	2021-05-05	1	NaN	1	1	1
12	2021-05-06	1	0.548088	2	1	1
13	2021-05-07	1	0.348813	1	1	1

SlidingWindow and SlidingWindowPanel are 2 useful functions that will allow you to create an array with segments of a pandas dataframe based on multiple criteria.

source

SlidingWindow

 SlidingWindow (window_len:int, stride:Optional[int]=1, start:int=0,
                pad_remainder:bool=False, padding:str='post',
                padding_value:float=nan, add_padding_feature:bool=True,
                get_x:Union[NoneType,int,list]=None,
                get_y:Union[NoneType,int,list]=None,
                y_func:Optional[<built-infunctioncallable>]=None,
                output_processor:Optional[<built-
                infunctioncallable>]=None, copy:bool=False,
                horizon:Union[int,list]=1, seq_first:bool=True,
                sort_by:Optional[list]=None, ascending:bool=True,
                check_leakage:bool=True)

Applies a sliding window to a 1d or 2d input (np.ndarray, torch.Tensor or pd.DataFrame)

Input:

    You can use np.ndarray, pd.DataFrame or torch.Tensor as input

    shape: (seq_len, ) or (seq_len, n_vars) if seq_first=True else (n_vars, seq_len)

	Type	Default	Details
window_len	int		length of lookback window
stride	Union[None, int]	1	n datapoints the window is moved ahead along the sequence. Default: 1. If None, stride=window_len (no overlap)
start	int	0	determines the step where the first window is applied: 0 (default) or a given step (int). Previous steps will be discarded.
pad_remainder	bool	False	allows to pad remainder subsequences when the sliding window is applied and get_y == [] (unlabeled data).
padding	str	post	‘pre’ or ‘post’ (optional, defaults to ‘pre’): pad either before or after each sequence. If pad_remainder == False, it indicates the starting point to create the sequence (‘pre’ from the end, and ‘post’ from the beginning)
padding_value	float	nan	value (float) that will be used for padding. Default: np.nan
add_padding_feature	bool	True	add an additional feature indicating whether each timestep is padded (1) or not (0).
get_x	Union[None, int, list]	None	indices of columns that contain the independent variable (xs). If None, all data will be used as x.
get_y	Union[None, int, list]	None	indices of columns that contain the target (ys). If None, all data will be used as y. [] means no y data is created (unlabeled data).
y_func	Optional[callable]	None	optional function to calculate the ys based on the get_y col/s and each y sub-window. y_func must be a function applied to axis=1!
output_processor	Optional[callable]	None	optional function to process the final output (X (and y if available)). This is useful when some values need to be removed.The function should take X and y (even if it’s None) as arguments.
copy	bool	False	copy the original object to avoid changes in it.
horizon	Union[int, list]	1	number of future datapoints to predict (y). If get_y is [] horizon will be set to 0.
seq_first	bool	True	True if input shape (seq_len, n_vars), False if input shape (n_vars, seq_len)
sort_by	Optional[list]	None	column/s used for sorting the array in ascending order
ascending	bool	True	used in sorting
check_leakage	bool	True	checks if there’s leakage in the output between X and y

wl = 5
stride = 5

t = np.repeat(np.arange(13).reshape(-1,1), 3, axis=-1)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=stride, pad_remainder=True, get_y=[])(t)
X

input shape: (13, 3)

array([[[ 0.,  1.,  2.,  3.,  4.],
        [ 0.,  1.,  2.,  3.,  4.],
        [ 0.,  1.,  2.,  3.,  4.],
        [ 0.,  0.,  0.,  0.,  0.]],

       [[ 5.,  6.,  7.,  8.,  9.],
        [ 5.,  6.,  7.,  8.,  9.],
        [ 5.,  6.,  7.,  8.,  9.],
        [ 0.,  0.,  0.,  0.,  0.]],

       [[10., 11., 12., nan, nan],
        [10., 11., 12., nan, nan],
        [10., 11., 12., nan, nan],
        [ 0.,  0.,  0.,  1.,  1.]]])

wl = 5
t = np.arange(10)
print('input shape:', t.shape)
X, y = SlidingWindow(wl)(t)
test_eq(X.shape[1:], (1, wl))
itemify(X,)

input shape: (10,)

(#5) [(array([[0, 1, 2, 3, 4]]),),(array([[1, 2, 3, 4, 5]]),),(array([[2, 3, 4, 5, 6]]),),(array([[3, 4, 5, 6, 7]]),),(array([[4, 5, 6, 7, 8]]),)]

wl = 5
h = 1

t = np.arange(10)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=1, horizon=h)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (10,)
[(array([[0, 1, 2, 3, 4]]), 5), (array([[1, 2, 3, 4, 5]]), 6), (array([[2, 3, 4, 5, 6]]), 7), (array([[3, 4, 5, 6, 7]]), 8), (array([[4, 5, 6, 7, 8]]), 9)]

wl = 5
h = 2 # 2 or more

t = np.arange(10)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, horizon=h)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, (2, ))

input shape: (10,)
[(array([[0, 1, 2, 3, 4]]), array([5, 6])), (array([[1, 2, 3, 4, 5]]), array([6, 7])), (array([[2, 3, 4, 5, 6]]), array([7, 8])), (array([[3, 4, 5, 6, 7]]), array([8, 9]))]

wl = 5
h = 2 # 2 or more

t = np.arange(10).reshape(1, -1)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=1, horizon=h, get_y=None, seq_first=False)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, (2, ))

input shape: (1, 10)
[(array([[0, 1, 2, 3, 4]]), array([5, 6])), (array([[1, 2, 3, 4, 5]]), array([6, 7])), (array([[2, 3, 4, 5, 6]]), array([7, 8])), (array([[3, 4, 5, 6, 7]]), array([8, 9]))]

wl = 5
h = 2 # 2 or more

t = np.arange(10).reshape(1, -1)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=1, horizon=h, seq_first=False)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))

input shape: (1, 10)
[(array([[0, 1, 2, 3, 4]]), array([5, 6])), (array([[1, 2, 3, 4, 5]]), array([6, 7])), (array([[2, 3, 4, 5, 6]]), array([7, 8])), (array([[3, 4, 5, 6, 7]]), array([8, 9]))]

wl = 5

t = np.arange(10).reshape(1, -1)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=3, horizon=1, get_y=None, seq_first=False)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (1, 10)
[(array([[0, 1, 2, 3, 4]]), 5), (array([[3, 4, 5, 6, 7]]), 8)]

wl = 5
start = 3

t = np.arange(20)
print('input shape:', t.shape)
X, y = SlidingWindow(wl, stride=None, horizon=1, start=start)(t)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (20,)
[(array([[3, 4, 5, 6, 7]]), 8), (array([[ 8,  9, 10, 11, 12]]), 13), (array([[13, 14, 15, 16, 17]]), 18)]

wl = 5

t = np.arange(20)
print('input shape:', t.shape)
df = pd.DataFrame(t, columns=['var'])
display(df)
X, y = SlidingWindow(wl, stride=None, horizon=1, get_y=None)(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (20,)
[(array([[0, 1, 2, 3, 4]]), 5), (array([[5, 6, 7, 8, 9]]), 10), (array([[10, 11, 12, 13, 14]]), 15)]

	var
0	0
1	1
2	2
3	3
4	4
5	5
6	6
7	7
8	8
9	9
10	10
11	11
12	12
13	13
14	14
15	15
16	16
17	17
18	18
19	19

wl = 5

t = np.arange(20)
print('input shape:', t.shape)
df = pd.DataFrame(t, columns=['var'])
display(df)
X, y = SlidingWindow(wl, stride=1, horizon=1, get_y=None)(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (20,)
[(array([[0, 1, 2, 3, 4]]), 5), (array([[1, 2, 3, 4, 5]]), 6), (array([[2, 3, 4, 5, 6]]), 7), (array([[3, 4, 5, 6, 7]]), 8), (array([[4, 5, 6, 7, 8]]), 9), (array([[5, 6, 7, 8, 9]]), 10), (array([[ 6,  7,  8,  9, 10]]), 11), (array([[ 7,  8,  9, 10, 11]]), 12), (array([[ 8,  9, 10, 11, 12]]), 13), (array([[ 9, 10, 11, 12, 13]]), 14), (array([[10, 11, 12, 13, 14]]), 15), (array([[11, 12, 13, 14, 15]]), 16), (array([[12, 13, 14, 15, 16]]), 17), (array([[13, 14, 15, 16, 17]]), 18), (array([[14, 15, 16, 17, 18]]), 19)]

	var
0	0
1	1
2	2
3	3
4	4
5	5
6	6
7	7
8	8
9	9
10	10
11	11
12	12
13	13
14	14
15	15
16	16
17	17
18	18
19	19

wl = 5

t = np.arange(20)
print('input shape:', t.shape)
df = pd.DataFrame(t, columns=['var']).T
display(df)
X, y = SlidingWindow(wl, stride=None, horizon=1, get_y=None, seq_first=False)(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (1, wl))
test_eq(items[0][1].shape, ())

input shape: (20,)
[(array([[0, 1, 2, 3, 4]]), 5), (array([[5, 6, 7, 8, 9]]), 10), (array([[10, 11, 12, 13, 14]]), 15)]

	0	1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17	18	19
var	0	1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17	18	19

wl = 5
n_vars = 3

t = (torch.stack(n_vars * [torch.arange(10)]).T * tensor([1, 10, 100]))
print('input shape:', t.shape)
df = pd.DataFrame(t, columns=[f'var_{i}' for i in range(n_vars)])
display(df)
X, y = SlidingWindow(wl, horizon=1)(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (n_vars, wl))

input shape: torch.Size([10, 3])
[(array([[  0,   1,   2,   3,   4],
       [  0,  10,  20,  30,  40],
       [  0, 100, 200, 300, 400]]), array([  5,  50, 500])), (array([[  1,   2,   3,   4,   5],
       [ 10,  20,  30,  40,  50],
       [100, 200, 300, 400, 500]]), array([  6,  60, 600])), (array([[  2,   3,   4,   5,   6],
       [ 20,  30,  40,  50,  60],
       [200, 300, 400, 500, 600]]), array([  7,  70, 700])), (array([[  3,   4,   5,   6,   7],
       [ 30,  40,  50,  60,  70],
       [300, 400, 500, 600, 700]]), array([  8,  80, 800])), (array([[  4,   5,   6,   7,   8],
       [ 40,  50,  60,  70,  80],
       [400, 500, 600, 700, 800]]), array([  9,  90, 900]))]

	var_0	var_1	var_2
0	0	0	0
1	1	10	100
2	2	20	200
3	3	30	300
4	4	40	400
5	5	50	500
6	6	60	600
7	7	70	700
8	8	80	800
9	9	90	900

wl = 5
n_vars = 3

t = (torch.stack(n_vars * [torch.arange(10)]).T * tensor([1, 10, 100]))
print('input shape:', t.shape)
df = pd.DataFrame(t, columns=[f'var_{i}' for i in range(n_vars)])
display(df)
X, y = SlidingWindow(wl, horizon=1, get_y="var_0")(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (n_vars, wl))

input shape: torch.Size([10, 3])
[(array([[  0,   1,   2,   3,   4],
       [  0,  10,  20,  30,  40],
       [  0, 100, 200, 300, 400]]), 5), (array([[  1,   2,   3,   4,   5],
       [ 10,  20,  30,  40,  50],
       [100, 200, 300, 400, 500]]), 6), (array([[  2,   3,   4,   5,   6],
       [ 20,  30,  40,  50,  60],
       [200, 300, 400, 500, 600]]), 7), (array([[  3,   4,   5,   6,   7],
       [ 30,  40,  50,  60,  70],
       [300, 400, 500, 600, 700]]), 8), (array([[  4,   5,   6,   7,   8],
       [ 40,  50,  60,  70,  80],
       [400, 500, 600, 700, 800]]), 9)]

	var_0	var_1	var_2
0	0	0	0
1	1	10	100
2	2	20	200
3	3	30	300
4	4	40	400
5	5	50	500
6	6	60	600
7	7	70	700
8	8	80	800
9	9	90	900

wl = 5
n_vars = 3

t = (torch.stack(n_vars * [torch.arange(10)]).T * tensor([1, 10, 100]))
print('input shape:', t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)]+['target']
df = pd.DataFrame(t, columns=columns)
display(df)
X, y = SlidingWindow(wl, horizon=1, get_x=columns[:-1], get_y='target')(df)
items = itemify(X, y)
print(items)
test_eq(items[0][0].shape, (n_vars-1, wl))
test_eq(items[0][1].shape, ())

input shape: torch.Size([10, 3])
[(array([[ 0,  1,  2,  3,  4],
       [ 0, 10, 20, 30, 40]]), 500), (array([[ 1,  2,  3,  4,  5],
       [10, 20, 30, 40, 50]]), 600), (array([[ 2,  3,  4,  5,  6],
       [20, 30, 40, 50, 60]]), 700), (array([[ 3,  4,  5,  6,  7],
       [30, 40, 50, 60, 70]]), 800), (array([[ 4,  5,  6,  7,  8],
       [40, 50, 60, 70, 80]]), 900)]

	var_0	var_1	target
0	0	0	0
1	1	10	100
2	2	20	200
3	3	30	300
4	4	40	400
5	5	50	500
6	6	60	600
7	7	70	700
8	8	80	800
9	9	90	900

n_vars = 3

t = (np.random.rand(1000, n_vars) - .5).cumsum(0)
print(t.shape)
plt.plot(t)
plt.show()
X, y = SlidingWindow(5, stride=None, horizon=0, get_x=[0,1], get_y=2)(t)
test_eq(X[0].shape, (n_vars-1, wl))
test_eq(y[0].shape, ())
print(X.shape, y.shape)

(1000, 3)
(200, 2, 5) (200,)

wl = 5
n_vars = 3

t = (np.random.rand(100, n_vars) - .5).cumsum(0)
print(t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)]+['target']
df = pd.DataFrame(t, columns=columns)
display(df)
X, y = SlidingWindow(5, horizon=0, get_x=columns[:-1], get_y='target')(df)
test_eq(X[0].shape, (n_vars-1, wl))
test_eq(y[0].shape, ())
print(X.shape, y.shape)

(100, 3)
(96, 2, 5) (96,)

	var_0	var_1	target
0	0.154072	0.197194	-0.083179
1	0.402744	-0.248788	-0.560573
2	0.448209	0.224215	-0.681264
3	0.631502	0.406760	-1.162043
4	1.099973	0.179926	-0.712690
...	...	...	...
95	-0.405079	3.662311	-2.779159
96	-0.445625	3.488809	-2.663381
97	-0.187349	3.304898	-2.695971
98	-0.100652	3.505663	-2.590652
99	0.371272	3.279901	-2.764369

100 rows × 3 columns

seq_len = 100
n_vars = 5
t = (np.random.rand(seq_len, n_vars) - .5).cumsum(0)
print(t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)]+['target']
df = pd.DataFrame(t, columns=columns)
display(df)
X, y = SlidingWindow(5, stride=1, horizon=0, get_x=columns[:-1], get_y='target', seq_first=True)(df)
test_eq(X[0].shape, (n_vars-1, wl))
test_eq(y[0].shape, ())
print(X.shape, y.shape)

(100, 5)
(96, 4, 5) (96,)

	var_0	var_1	var_2	var_3	target
0	0.443639	-0.288128	-0.049732	0.288915	0.325872
1	-0.047608	-0.009738	0.056768	0.541395	0.017496
2	-0.243972	0.102227	0.361387	0.628397	0.049012
3	-0.721266	0.045104	0.724062	0.940693	0.510875
4	-0.641269	0.141927	0.793837	1.158903	0.417040
...	...	...	...	...	...
95	3.488117	2.345512	0.745483	0.258568	2.468550
96	3.187006	1.945844	0.833228	0.511198	2.115330
97	3.019862	1.739802	0.488732	0.881324	2.387837
98	3.314247	1.992000	0.119230	0.797794	2.327720
99	3.394578	2.012458	0.003244	0.387125	2.345970

100 rows × 5 columns

seq_len = 100
n_vars = 5

t = (np.random.rand(seq_len, n_vars) - .5).cumsum(0)
print(t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)] + ['target']
df = pd.DataFrame(t, columns=columns).T
display(df)
X, y = SlidingWindow(5, stride=1, horizon=0, get_x=columns[:-1], get_y='target', seq_first=False)(df)
test_eq(X[0].shape, (n_vars-1, wl))
test_eq(y[0].shape, ())
print(X.shape, y.shape)

(100, 5)
(96, 4, 5) (96,)

	0	1	2	3	4	5	6	7	8	9	...	90	91	92	93	94	95	96	97	98	99
var_0	-0.407162	-0.742169	-1.193053	-1.058644	-0.721243	-1.056788	-1.316226	-1.247859	-1.391482	-1.258618	...	-2.847911	-3.118643	-3.444248	-3.036050	-2.664068	-2.473782	-2.508080	-2.878210	-2.841170	-2.688932
var_1	0.111643	-0.286318	-0.221917	-0.026094	-0.332200	-0.376518	-0.144763	0.225361	0.487134	0.435856	...	1.569158	1.294548	1.564455	1.501243	1.490928	1.450602	1.440730	1.755607	1.380986	1.236284
var_2	-0.126951	-0.484267	-0.480375	-0.706987	-0.571379	-0.561959	-0.717696	-0.586035	-0.298053	-0.047405	...	-1.748096	-1.508691	-1.158258	-1.116485	-1.153738	-1.575450	-1.875091	-1.613255	-1.274859	-1.592096
var_3	-0.462238	-0.748774	-0.625473	-0.360442	-0.789178	-0.530832	-0.785290	-0.413452	0.083685	-0.110964	...	-4.873450	-4.382297	-4.531454	-4.087051	-4.087801	-4.391084	-4.262526	-4.650170	-4.465874	-4.535273
target	0.241454	0.084139	-0.012974	0.096328	0.501035	0.697043	0.229185	0.497430	0.552922	0.218345	...	-4.582426	-4.194067	-3.785398	-3.808516	-3.629740	-3.398645	-3.828007	-3.600028	-3.614195	-3.592783

5 rows × 100 columns

seq_len = 100
n_vars = 5
t = (np.random.rand(seq_len, n_vars) - .5).cumsum(0)
print(t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)] + ['target']
df = pd.DataFrame(t, columns=columns).T
display(df)
X, y = SlidingWindow(5, stride=None, horizon=0, get_x=columns[:-1], get_y='target', seq_first=False)(df)
test_eq(X[0].shape, (n_vars-1, wl))
test_eq(y[0].shape, ())
print(X.shape, y.shape)

(100, 5)
(20, 4, 5) (20,)

	0	1	2	3	4	5	6	7	8	9	...	90	91	92	93	94	95	96	97	98	99
var_0	0.210943	-0.264863	-0.307942	0.176782	-0.188244	0.118824	0.593353	0.611408	0.176396	0.566034	...	-4.738294	-5.138743	-5.203979	-4.835758	-4.534974	-4.310112	-4.366365	-4.328250	-4.527717	-4.432726
var_1	-0.086375	-0.457413	0.025571	0.428256	0.611573	0.319714	-0.085129	0.161735	0.052730	-0.356617	...	7.203539	7.300534	7.267954	6.838923	7.054134	6.612532	7.108269	6.966000	7.407915	7.332567
var_2	0.166139	-0.231839	-0.468804	-0.565628	-0.500941	-0.706951	-0.881385	-1.138549	-0.978276	-0.952727	...	0.391942	0.802356	0.395688	0.033288	0.147283	0.589911	0.360847	0.322019	0.478120	0.278228
var_3	-0.234297	-0.467480	-0.925036	-0.572783	-0.345585	0.149537	-0.078098	-0.577732	-0.771975	-0.322283	...	-1.487032	-1.971348	-2.300616	-2.767312	-2.657974	-2.880908	-2.567235	-2.758240	-2.605518	-2.166444
target	-0.416187	-0.164800	-0.283554	-0.534897	-0.896808	-0.456572	-0.889556	-1.178456	-0.877891	-1.176442	...	-6.094650	-6.510793	-6.408799	-6.685696	-6.672726	-6.210781	-6.377436	-5.974001	-5.755187	-5.608240

5 rows × 100 columns

from tsai.data.validation import TrainValidTestSplitter

seq_len = 100
n_vars = 5
t = (np.random.rand(seq_len, n_vars) - .5).cumsum(0)
print(t.shape)
columns=[f'var_{i}' for i in range(n_vars-1)]+['target']
df = pd.DataFrame(t, columns=columns)
display(df)
X, y = SlidingWindow(5, stride=1, horizon=0, get_x=columns[:-1], get_y='target', seq_first=True)(df)
splits = TrainValidTestSplitter(valid_size=.2, shuffle=False)(y)
X.shape, y.shape, splits

(100, 5)

	var_0	var_1	var_2	var_3	target
0	0.123248	-0.081596	0.099444	0.447980	-0.397975
1	0.469671	-0.334499	0.307867	0.141345	-0.131085
2	0.522902	-0.696817	0.386597	0.156818	0.128043
3	0.487025	-0.966153	-0.050574	-0.248479	-0.088962
4	0.396284	-1.319821	-0.113121	-0.379227	0.313690
...	...	...	...	...	...
95	6.138836	-1.602917	1.713049	1.421797	-1.873899
96	5.892472	-1.896914	1.401137	1.065859	-2.239942
97	5.421917	-1.728568	1.481270	0.998533	-2.157474
98	5.763120	-1.404330	1.931361	1.295956	-1.934397
99	5.827842	-1.762438	1.831712	1.014259	-1.831573

100 rows × 5 columns

((96, 4, 5),
 (96,),
 ((#77) [0,1,2,3,4,5,6,7,8,9...], (#19) [77,78,79,80,81,82,83,84,85,86...]))

data = np.concatenate([np.linspace(0, 1, 11).reshape(-1,1).repeat(2, 1), np.arange(11).reshape(-1,1)], -1)
df_test = pd.DataFrame(data, columns=['col1', 'col2', 'target'])
df_test['target'] = df_test['target'].astype(int)
df_test

	col1	col2	target
0	0.0	0.0	0
1	0.1	0.1	1
2	0.2	0.2	2
3	0.3	0.3	3
4	0.4	0.4	4
5	0.5	0.5	5
6	0.6	0.6	6
7	0.7	0.7	7
8	0.8	0.8	8
9	0.9	0.9	9
10	1.0	1.0	10

def _y_func(o): return o[:, 0]

for wl in np.arange(1, 20):
    x, y = SlidingWindow(wl, None, pad_remainder=True, get_x=['col1', 'col2'], get_y=['target'], horizon=-wl, y_func=_y_func)(df_test)
    test_eq(x.shape[0], math.ceil((len(df_test))/wl))
    test_eq(x.shape[0], y.shape[0])
    test_eq(x.shape[2], wl)
    test_close(x[:, 0, 0]*10, y)

for wl in np.arange(1, 20):
    x, y = SlidingWindow(wl, None, pad_remainder=True, get_x=['col1', 'col2'], get_y=['target'], horizon=-wl, y_func=None)(df_test)
    test_eq(x.shape[0], math.ceil((len(df_test))/ wl))
    test_eq(x.shape[0], y.shape[0])
    test_eq(x.shape[2], wl)

for wl in np.arange(1, len(df_test)+1):
    x, y = SlidingWindow(wl, None, pad_remainder=False, get_x=['col1', 'col2'], get_y=['target'], horizon=-wl, y_func=None)(df_test)
    test_eq(x.shape[0], len(df_test) // wl)
    test_eq(x.shape[0], y.shape[0])
    test_eq(x.shape[2], wl)

for wl in np.arange(1, 20):
    x, _ = SlidingWindow(wl, None, pad_remainder=True, get_x=['col1', 'col2'], get_y=[], horizon=0)(df_test)
    test_eq(x.shape[0], math.ceil((len(df_test))/wl))
    test_eq(x.shape[2], wl)

for wl in np.arange(2, len(df_test)):
    x, _ = SlidingWindow(wl, wl, pad_remainder=False, get_x=['col1', 'col2'], get_y=[], horizon=0)(df_test)
    test_eq(x.shape[0], len(df_test) // wl)
    test_eq(x.shape[2], wl)

df = pd.DataFrame()
df['sample_id'] = np.concatenate([np.ones(n)*(i + 1) for i,n in enumerate([13])])
df['var1'] = df['sample_id'] + df.index.values - 1
df['var2'] = df['var1'] * 10
df['target'] = (df['var1']).astype(int)
df['sample_id'] = df['sample_id'].astype(int)
df

	sample_id	var1	var2	target
0	1	0.0	0.0	0
1	1	1.0	10.0	1
2	1	2.0	20.0	2
3	1	3.0	30.0	3
4	1	4.0	40.0	4
5	1	5.0	50.0	5
6	1	6.0	60.0	6
7	1	7.0	70.0	7
8	1	8.0	80.0	8
9	1	9.0	90.0	9
10	1	10.0	100.0	10
11	1	11.0	110.0	11
12	1	12.0	120.0	12

X, y = SlidingWindow(window_len=3, stride=2, start=3, pad_remainder=False, padding="pre", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (2, 2, 3))
test_eq(y.shape, (2, 4))
X, y

(array([[[ 4.,  5.,  6.],
         [40., 50., 60.]],
 
        [[ 6.,  7.,  8.],
         [60., 70., 80.]]]),
 array([[ 7,  8,  9, 10],
        [ 9, 10, 11, 12]]))

X, y = SlidingWindow(window_len=3, stride=2, start=3, pad_remainder=True, padding="pre", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (3, 2, 3))
test_eq(y.shape, (3, 4))
X, y

(array([[[nan,  3.,  4.],
         [nan, 30., 40.]],
 
        [[ 4.,  5.,  6.],
         [40., 50., 60.]],
 
        [[ 6.,  7.,  8.],
         [60., 70., 80.]]]),
 array([[ 5,  6,  7,  8],
        [ 7,  8,  9, 10],
        [ 9, 10, 11, 12]]))

X, y = SlidingWindow(window_len=3, stride=2, start=3, pad_remainder=False, padding="post", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (2, 2, 3))
test_eq(y.shape, (2, 4))
X, y

(array([[[ 3.,  4.,  5.],
         [30., 40., 50.]],
 
        [[ 5.,  6.,  7.],
         [50., 60., 70.]]]),
 array([[ 6,  7,  8,  9],
        [ 8,  9, 10, 11]]))

X, y = SlidingWindow(window_len=3, stride=2, start=3, pad_remainder=True, padding="post", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (3, 2, 3))
test_eq(y.shape, (3, 4))
X, y

(array([[[ 3.,  4.,  5.],
         [30., 40., 50.]],
 
        [[ 5.,  6.,  7.],
         [50., 60., 70.]],
 
        [[ 7.,  8.,  9.],
         [70., 80., 90.]]]),
 array([[ 6.,  7.,  8.,  9.],
        [ 8.,  9., 10., 11.],
        [10., 11., 12., nan]]))

X, y = SlidingWindow(window_len=10, stride=2, start=3, pad_remainder=True, padding="pre", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (1, 2, 10))
test_eq(y.shape, (1, 4))
X, y

(array([[[nan, nan, nan, nan,  3.,  4.,  5.,  6.,  7.,  8.],
         [nan, nan, nan, nan, 30., 40., 50., 60., 70., 80.]]]),
 array([[ 9, 10, 11, 12]]))

X, y = SlidingWindow(window_len=10, stride=2, start=3, pad_remainder=True, padding="post", padding_value=np.nan, add_padding_feature=False,
                     get_x=["var1", "var2"], get_y=["target"], y_func=None, output_processor=None, copy=False, horizon=4, seq_first=True, sort_by=None,
                     ascending=True, check_leakage=True)(df)
test_eq(X.shape, (1, 2, 10))
test_eq(y.shape, (1, 4))
X, y

(array([[[  3.,   4.,   5.,   6.,   7.,   8.,   9.,  10.,  11.,  12.],
         [ 30.,  40.,  50.,  60.,  70.,  80.,  90., 100., 110., 120.]]]),
 array([[nan, nan, nan, nan]]))

source

SlidingWindowPanel

 SlidingWindowPanel (window_len:int, unique_id_cols:list,
                     stride:Optional[int]=1, start:int=0,
                     pad_remainder:bool=False, padding:str='post',
                     padding_value:float=nan,
                     add_padding_feature:bool=True,
                     get_x:Union[NoneType,int,list]=None,
                     get_y:Union[NoneType,int,list]=None,
                     y_func:Optional[<built-infunctioncallable>]=None,
                     output_processor:Optional[<built-
                     infunctioncallable>]=None, copy:bool=False,
                     horizon:Union[int,list]=1, seq_first:bool=True,
                     sort_by:Optional[list]=None, ascending:bool=True,
                     check_leakage:bool=True, return_key:bool=False,
                     verbose:bool=True)

Applies a sliding window to a pd.DataFrame.

Args: window_len = length of lookback window unique_id_cols = pd.DataFrame columns that will be used to identify a time series for each entity. stride = n datapoints the window is moved ahead along the sequence. Default: 1. If None, stride=window_len (no overlap) start = determines the step where the first window is applied: 0 (default), a given step (int), or random within the 1st stride (None). pad_remainder = allows to pad remainder subsequences when the sliding window is applied and get_y == [] (unlabeled data). padding = ‘pre’ or ‘post’ (optional, defaults to ‘pre’): pad either before or after each sequence. If pad_remainder == False, it indicates the starting point to create the sequence (‘pre’ from the end, and ‘post’ from the beginning) padding_value = value (float) that will be used for padding. Default: np.nan add_padding_feature = add an additional feature indicating whether each timestep is padded (1) or not (0). horizon = number of future datapoints to predict (y). If get_y is [] horizon will be set to 0. * 0 for last step in each sub-window. * n > 0 for a range of n future steps (1 to n). * n < 0 for a range of n past steps (-n + 1 to 0). * list : for those exact timesteps. get_x = indices of columns that contain the independent variable (xs). If None, all data will be used as x. get_y = indices of columns that contain the target (ys). If None, all data will be used as y. [] means no y data is created (unlabeled data). y_func = function to calculate the ys based on the get_y col/s and each y sub-window. y_func must be a function applied to axis=1! output_processor = optional function to filter output (X (and y if available)). This is useful when some values need to be removed. The function should take X and y (even if it’s None) as arguments. copy = copy the original object to avoid changes in it. seq_first = True if input shape (seq_len, n_vars), False if input shape (n_vars, seq_len) sort_by = column/s used for sorting the array in ascending order ascending = used in sorting check_leakage = checks if there’s leakage in the output between X and y return_key = when True, the key corresponsing to unique_id_cols for each sample is returned verbose = controls verbosity. True or 1 displays progress bar. 2 or more show records that cannot be created due to its length.

Input: You can use np.ndarray, pd.DataFrame or torch.Tensor as input shape: (seq_len, ) or (seq_len, n_vars) if seq_first=True else (n_vars, seq_len)

samples = 100_000
wl = 5
n_vars = 10

t = (torch.stack(n_vars * [torch.arange(samples)]).T * tensor([10**i for i in range(n_vars)]))
df = pd.DataFrame(t, columns=[f'var_{i}' for i in range(n_vars)])
df['time'] = np.arange(len(t))
df['device'] = 0
df['target'] = np.random.randint(0, 2, len(df))
df2 = df.copy()
df3 = df.copy()
cols = ['var_0', 'var_1', 'var_2', 'device', 'target']
df2[cols] = df2[cols] + 1
df3[cols] = df3[cols] + 2
df2 = df2.loc[:3]
df['region'] = 'A'
df2['region'] = 'A'
df3['region'] = 'B'
df = pd.concat([df, df2, df3], ignore_index=True)
df['index'] = np.arange(len(df))
df = df.sample(frac=1).reset_index(drop=True)
display(df.head())
df.shape

	var_0	var_1	var_2	var_3	var_4	var_5	var_6	var_7	var_8	var_9	time	device	target	region	index
0	86008	860080	8600800	86008000	860080000	8600800000	86008000000	860080000000	8600800000000	86008000000000	86008	0	0	A	86008
1	90003	900012	9000102	90001000	900010000	9000100000	90001000000	900010000000	9000100000000	90001000000000	90001	2	2	B	190005
2	43819	438172	4381702	43817000	438170000	4381700000	43817000000	438170000000	4381700000000	43817000000000	43817	2	3	B	143821
3	80751	807492	8074902	80749000	807490000	8074900000	80749000000	807490000000	8074900000000	80749000000000	80749	2	3	B	180753
4	84917	849152	8491502	84915000	849150000	8491500000	84915000000	849150000000	8491500000000	84915000000000	84915	2	3	B	184919

(200004, 15)

X, y = SlidingWindowPanel(window_len=5, unique_id_cols=['device'], stride=1, start=0, get_x=df.columns[:n_vars], get_y=['target'], 
                          horizon=0, seq_first=True, sort_by=['time'], ascending=True, return_key=False)(df)
X.shape, y.shape

processing data...
...data processed
concatenating X...
...X concatenated
concatenating y...
...y concatenated

((199992, 10, 5), (199992,))

X, y, key = SlidingWindowPanel(window_len=5, unique_id_cols=['device'], stride=1, start=0, get_x=df.columns[:n_vars], get_y=['target'], 
                               horizon=0, seq_first=True, sort_by=['time'], ascending=True, return_key=True)(df)
X.shape, y.shape, key.shape

processing data...
...data processed
concatenating X...
...X concatenated
concatenating y...
...y concatenated

((199992, 10, 5), (199992,), (199992,))

X, y = SlidingWindowPanel(window_len=5, unique_id_cols=['device', 'region'], stride=1, start=0, get_x=df.columns[:n_vars], get_y=['target'], 
                          horizon=0, seq_first=True, sort_by=['time'], ascending=True)(df)
X.shape, y.shape

processing data...
...data processed
concatenating X...
...X concatenated
concatenating y...
...y concatenated

((199992, 10, 5), (199992,))

# y_func must be a function applied to axis=1!
def y_max(o): return np.max(o, axis=1)

X, y = SlidingWindowPanel(window_len=5, unique_id_cols=['device', 'region'], stride=1, start=0, get_x=df.columns[:n_vars], get_y=['target'], 
                          y_func=y_max, horizon=5, seq_first=True, sort_by=['time'], ascending=True)(df)
X.shape, y.shape

processing data...
...data processed
concatenating X...
...X concatenated
concatenating y...
...y concatenated

0.00% [0/3 00:00<?]

((199982, 10, 5), (199982,))

source

identify_padding

 identify_padding (float_mask, value=-1)

Identifies padded subsequences in a mask of type float

This function identifies as padded subsequences those where all values == nan from the end of the sequence (last dimension) across all channels, and sets those values to the selected value (default = -1)

Args: mask: boolean or float mask value: scalar that will be used to identify padded subsequences

wl = 5
stride = 5

t = np.repeat(np.arange(13).reshape(-1,1), 3, axis=-1)
print('input shape:', t.shape)
X, _ = SlidingWindow(wl, stride=stride, pad_remainder=True, get_y=[])(t)
X = tensor(X)
X[0, 1, -2:] = np.nan
X[1,..., :3] = np.nan
print(X)
identify_padding(torch.isnan(X).float())

input shape: (13, 3)
tensor([[[ 0.,  1.,  2.,  3.,  4.],
         [ 0.,  1.,  2., nan, nan],
         [ 0.,  1.,  2.,  3.,  4.],
         [ 0.,  0.,  0.,  0.,  0.]],

        [[nan, nan, nan,  8.,  9.],
         [nan, nan, nan,  8.,  9.],
         [nan, nan, nan,  8.,  9.],
         [nan, nan, nan,  0.,  0.]],

        [[10., 11., 12., nan, nan],
         [10., 11., 12., nan, nan],
         [10., 11., 12., nan, nan],
         [ 0.,  0.,  0.,  1.,  1.]]])

tensor([[[0., 0., 0., 0., 0.],
         [0., 0., 0., 1., 1.],
         [0., 0., 0., 0., 0.],
         [0., 0., 0., 0., 0.]],

        [[1., 1., 1., 0., 0.],
         [1., 1., 1., 0., 0.],
         [1., 1., 1., 0., 0.],
         [1., 1., 1., 0., 0.]],

        [[0., 0., 0., 1., 1.],
         [0., 0., 0., 1., 1.],
         [0., 0., 0., 1., 1.],
         [0., 0., 0., 0., 0.]]])

Forecasting data preparation

source

basic_data_preparation_fn

 basic_data_preparation_fn (df, drop_duplicates=True, datetime_col=None,
                            use_index=False, keep='last',
                            add_missing_datetimes=True, freq='1D',
                            method=None, sort_by=None)

	Type	Default	Details
df			dataframe to preprocess
drop_duplicates	bool	True	flag to indicate if rows with duplicate datetime info should be removed
datetime_col	NoneType	None	str indicating the name of the column/s that contains the datetime info
use_index	bool	False	flag to indicate if the datetime info is in the index
keep	str	last	str to indicate what data should be kept in case of duplicate rows
add_missing_datetimes	bool	True	flaf to indicate if missing datetimes should be added
freq	str	1D	str to indicate the frequency used in the datetime info. Used in case missing timestamps exists
method	NoneType	None	str indicating the method used to fill data for missing timestamps: None, ‘bfill’, ‘ffill’
sort_by	NoneType	None	str or list of str to indicate if how to sort data. If use_index=True the index will be used to sort the dataframe.

df_len = 100
datetime_col = 'datetime' 
df = pd.DataFrame(np.arange(df_len), columns=['value'])
df['datetime'] = pd.date_range(pd.to_datetime('1749-03-31'), periods=df_len, freq='1D')
df['type'] = 1
# drop 10 rows at random
df = df.drop(df.sample(10).index)
# add 2 duplicated rows
df = pd.concat([df, df.sample(2)])
display(df)

new_df = basic_data_preparation_fn(df, drop_duplicates=True, datetime_col=datetime_col, use_index=False, keep='last', 
                                   add_missing_datetimes=True, freq='1D', method='ffill', sort_by=datetime_col)
display(new_df)

	value	datetime	type
0	0	1749-03-31	1
1	1	1749-04-01	1
3	3	1749-04-03	1
4	4	1749-04-04	1
5	5	1749-04-05	1
...	...	...	...
96	96	1749-07-05	1
97	97	1749-07-06	1
99	99	1749-07-08	1
0	0	1749-03-31	1
19	19	1749-04-19	1

92 rows × 3 columns

	value	datetime	type
0	0	1749-03-31	1
1	1	1749-04-01	1
2	1	1749-04-02	1
3	3	1749-04-03	1
4	4	1749-04-04	1
...	...	...	...
95	95	1749-07-04	1
96	96	1749-07-05	1
97	97	1749-07-06	1
98	97	1749-07-07	1
99	99	1749-07-08	1

100 rows × 3 columns

source

check_safe_conversion

 check_safe_conversion (o, dtype='float32', cols=None)

Checks if the conversion to float is safe

assert check_safe_conversion(-2**11, 'float16') == True
assert check_safe_conversion(-2**11 - 1, 'float16') == False
assert check_safe_conversion(2**24, 'float32') == True
assert check_safe_conversion(2**24+1, 'float32') == False
assert check_safe_conversion(2**53, 'float64') == True
assert check_safe_conversion(2**53+1, 'float64') == False

df = pd.DataFrame({'a': [1, 2, 3], 'b': [2**24, 2**24+1, 2**24+2]})
assert not check_safe_conversion(df, 'float32')
assert check_safe_conversion(df, 'int32')
assert check_safe_conversion(df, 'float32', cols='a')
assert not check_safe_conversion(df, 'float32', cols='b')

-2147483648 1 3 2147483647
-2147483648 16777216 16777218 2147483647

/var/folders/42/4hhwknbd5kzcbq48tmy_gbp00000gn/T/ipykernel_30986/657350933.py:39: UserWarning: Unsafe conversion to float32: {'a': True, 'b': False}
  warnings.warn(f"Unsafe conversion to {dtype}: {dict(zip(cols, checks))}")
/var/folders/42/4hhwknbd5kzcbq48tmy_gbp00000gn/T/ipykernel_30986/657350933.py:39: UserWarning: Unsafe conversion to float32: {'b': False}
  warnings.warn(f"Unsafe conversion to {dtype}: {dict(zip(cols, checks))}")

source

prepare_forecasting_data

from tsai.data.validation import get_forecasting_splits

fcst_history = 10 
fcst_horizon = 5
stride = 1
valid_size=0.2
test_size=0.2

df = pd.DataFrame()
df['target'] = np.arange(50)

X, y = prepare_forecasting_data(df, fcst_history, fcst_horizon)
splits = get_forecasting_splits(df, fcst_history, fcst_horizon, valid_size=valid_size, test_size=test_size, stride=stride, show_plot=False)
assert y[splits[0]][-1][0][-1] == y[splits[1]][0][0][0] - stride
assert y[splits[1]][-1][0][-1] == y[splits[2]][0][0][0] - stride
for s,t in zip(splits, ['\ntrain_split:', '\nvalid_split:', '\ntest_split :']):
    print(t)
    for xi, yi in zip(X[s], y[s]):
        print(xi, yi)


train_split:
[[0 1 2 3 4 5 6 7 8 9]] [[10 11 12 13 14]]
[[ 1  2  3  4  5  6  7  8  9 10]] [[11 12 13 14 15]]
[[ 2  3  4  5  6  7  8  9 10 11]] [[12 13 14 15 16]]
[[ 3  4  5  6  7  8  9 10 11 12]] [[13 14 15 16 17]]
[[ 4  5  6  7  8  9 10 11 12 13]] [[14 15 16 17 18]]
[[ 5  6  7  8  9 10 11 12 13 14]] [[15 16 17 18 19]]
[[ 6  7  8  9 10 11 12 13 14 15]] [[16 17 18 19 20]]
[[ 7  8  9 10 11 12 13 14 15 16]] [[17 18 19 20 21]]
[[ 8  9 10 11 12 13 14 15 16 17]] [[18 19 20 21 22]]
[[ 9 10 11 12 13 14 15 16 17 18]] [[19 20 21 22 23]]
[[10 11 12 13 14 15 16 17 18 19]] [[20 21 22 23 24]]
[[11 12 13 14 15 16 17 18 19 20]] [[21 22 23 24 25]]
[[12 13 14 15 16 17 18 19 20 21]] [[22 23 24 25 26]]
[[13 14 15 16 17 18 19 20 21 22]] [[23 24 25 26 27]]
[[14 15 16 17 18 19 20 21 22 23]] [[24 25 26 27 28]]
[[15 16 17 18 19 20 21 22 23 24]] [[25 26 27 28 29]]

valid_split:
[[20 21 22 23 24 25 26 27 28 29]] [[30 31 32 33 34]]
[[21 22 23 24 25 26 27 28 29 30]] [[31 32 33 34 35]]
[[22 23 24 25 26 27 28 29 30 31]] [[32 33 34 35 36]]
[[23 24 25 26 27 28 29 30 31 32]] [[33 34 35 36 37]]
[[24 25 26 27 28 29 30 31 32 33]] [[34 35 36 37 38]]
[[25 26 27 28 29 30 31 32 33 34]] [[35 36 37 38 39]]

test_split :
[[30 31 32 33 34 35 36 37 38 39]] [[40 41 42 43 44]]
[[31 32 33 34 35 36 37 38 39 40]] [[41 42 43 44 45]]
[[32 33 34 35 36 37 38 39 40 41]] [[42 43 44 45 46]]
[[33 34 35 36 37 38 39 40 41 42]] [[43 44 45 46 47]]
[[34 35 36 37 38 39 40 41 42 43]] [[44 45 46 47 48]]
[[35 36 37 38 39 40 41 42 43 44]] [[45 46 47 48 49]]

fcst_history = 10 
fcst_horizon = 5
stride = 1
valid_size=0.2
test_size=0.2

df = pd.DataFrame()
df['target'] = np.arange(50)

X, y = prepare_forecasting_data(df, fcst_history, fcst_horizon, x_vars=None, y_vars=[])
splits = get_forecasting_splits(df, fcst_history, fcst_horizon, valid_size=valid_size, test_size=test_size, stride=stride, show_plot=False)
assert y is None

df_len = 100
n_values = 3
datetime_col = 'datetime' 
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**i).astype(np.float32)
display(df)

fcst_history = 10
fcst_horizon = 5
x_vars = df.columns
y_vars = None
dtype = None

X, y = prepare_forecasting_data(df, fcst_history=fcst_history, fcst_horizon=fcst_horizon, x_vars=x_vars, y_vars=y_vars, dtype=dtype)
test_eq(X.shape, (86, 3, 10))
test_eq(y.shape, (86, 3, 5))
test_eq(y[:3, :, 0],  X[:3, :, -1] + np.array([1, 10, 100]).reshape(1, 1, -1))
print(X[:3].astype(int))
print(y[:3].astype(int))

	value_0	value_1	value_2
0	0.0	0.0	0.0
1	1.0	10.0	100.0
2	2.0	20.0	200.0
3	3.0	30.0	300.0
4	4.0	40.0	400.0
...	...	...	...
95	95.0	950.0	9500.0
96	96.0	960.0	9600.0
97	97.0	970.0	9700.0
98	98.0	980.0	9800.0
99	99.0	990.0	9900.0

100 rows × 3 columns

[[[   0    1    2    3    4    5    6    7    8    9]
  [   0   10   20   30   40   50   60   70   80   90]
  [   0  100  200  300  400  500  600  700  800  900]]

 [[   1    2    3    4    5    6    7    8    9   10]
  [  10   20   30   40   50   60   70   80   90  100]
  [ 100  200  300  400  500  600  700  800  900 1000]]

 [[   2    3    4    5    6    7    8    9   10   11]
  [  20   30   40   50   60   70   80   90  100  110]
  [ 200  300  400  500  600  700  800  900 1000 1100]]]
[[[  10   11   12   13   14]
  [ 100  110  120  130  140]
  [1000 1100 1200 1300 1400]]

 [[  11   12   13   14   15]
  [ 110  120  130  140  150]
  [1100 1200 1300 1400 1500]]

 [[  12   13   14   15   16]
  [ 120  130  140  150  160]
  [1200 1300 1400 1500 1600]]]

df_len = 100
n_values = 3
datetime_col = 'datetime' 
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)

df['datetime'] = pd.date_range(pd.to_datetime('1749-03-31'), periods=df_len, freq='1D')
df['type'] = np.random.randint(0, 4, df_len)
df['target'] = np.arange(df_len)
display(df)

fcst_history = 10
fcst_horizon = 5
x_vars = ['value_0', 'value_1', 'value_2', 'target']
y_vars = 'target'
dtype = np.float32

X, y = prepare_forecasting_data(df, fcst_history=fcst_history, fcst_horizon=fcst_horizon, x_vars=x_vars, y_vars=y_vars, dtype=dtype)
test_eq(X.shape, (86, 4, 10))
test_eq(y.shape, (86, 1, 5))
print(X[:3].astype(int))
print(y[:3])

	value_0	value_1	value_2	datetime	type	target
0	0.0	0.0	0.0	1749-03-31	3	0
1	10.0	100.0	1000.0	1749-04-01	1	1
2	20.0	200.0	2000.0	1749-04-02	1	2
3	30.0	300.0	3000.0	1749-04-03	1	3
4	40.0	400.0	4000.0	1749-04-04	2	4
...	...	...	...	...	...	...
95	950.0	9500.0	95000.0	1749-07-04	0	95
96	960.0	9600.0	96000.0	1749-07-05	0	96
97	970.0	9700.0	97000.0	1749-07-06	3	97
98	980.0	9800.0	98000.0	1749-07-07	2	98
99	990.0	9900.0	99000.0	1749-07-08	1	99

100 rows × 6 columns

[[[    0    10    20    30    40    50    60    70    80    90]
  [    0   100   200   300   400   500   600   700   800   900]
  [    0  1000  2000  3000  4000  5000  6000  7000  8000  9000]
  [    0     1     2     3     4     5     6     7     8     9]]

 [[   10    20    30    40    50    60    70    80    90   100]
  [  100   200   300   400   500   600   700   800   900  1000]
  [ 1000  2000  3000  4000  5000  6000  7000  8000  9000 10000]
  [    1     2     3     4     5     6     7     8     9    10]]

 [[   20    30    40    50    60    70    80    90   100   110]
  [  200   300   400   500   600   700   800   900  1000  1100]
  [ 2000  3000  4000  5000  6000  7000  8000  9000 10000 11000]
  [    2     3     4     5     6     7     8     9    10    11]]]
[[[10. 11. 12. 13. 14.]]

 [[11. 12. 13. 14. 15.]]

 [[12. 13. 14. 15. 16.]]]

source

get_today

 get_today (datetime_format='%Y-%m-%d')

test_eq(get_today(), dt.datetime.today().strftime("%Y-%m-%d"))

source

split_fcst_datetime

 split_fcst_datetime (fcst_datetime)

Define fcst start and end dates

	Details
fcst_datetime	str or list of str with datetime

test_eq(split_fcst_datetime(None), (None, None))
test_eq(split_fcst_datetime('2020-01-01'), ('2020-01-01', '2020-01-01'))
test_eq(split_fcst_datetime(['2019-01-01', '2020-01-01']), ['2019-01-01', '2020-01-01'])

source

set_df_datetime

 set_df_datetime (df, datetime_col=None, use_index=False)

Make sure datetime column or index is of the right date type.

# Test
df_len = 100
n_values = 3
datetime_col = 'datetime'
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)
df['datetime'] = pd.date_range(pd.to_datetime('1749-03-31'), periods=df_len, freq='1D')
set_df_datetime(df, datetime_col=datetime_col)
test_eq(df['datetime'].dtypes, np.dtype('datetime64[ns]'))
df_index = df.set_index('datetime')
set_df_datetime(df_index, use_index=True)
test_eq(df_index.index.dtype, np.dtype('datetime64[ns]'))

source

get_df_datetime_bounds

 get_df_datetime_bounds (df, datetime_col=None, use_index=False)

Returns the start date and and dates used by the forecast

	Type	Default	Details
df			dataframe containing forecasting data
datetime_col	NoneType	None	str data column containing the datetime
use_index	bool	False	bool flag to indicate if index should be used to get column

# Test
df_len = 100
n_values = 3
datetime_col = 'datetime'
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)
df['datetime'] = pd.date_range(pd.to_datetime('1749-03-31'), periods=df_len, freq='1D')
test_eq(get_df_datetime_bounds(df, datetime_col=datetime_col), (df['datetime'].min(), df['datetime'].max()))
df_index = df.set_index('datetime')
test_eq(get_df_datetime_bounds(df_index, use_index=True), (df_index.index.min(), df_index.index.max()))

source

get_fcst_bounds

 get_fcst_bounds (df, fcst_datetime, fcst_history=None, fcst_horizon=None,
                  freq='D', datetime_format='%Y-%m-%d', datetime_col=None,
                  use_index=False)

Returns the start and end datetimes used by the forecast

	Type	Default	Details
df			dataframe containing forecasting data
fcst_datetime			datetime for which a fcst is created. Optionally tuple of datatimes if the fcst is created for a range of dates.
fcst_history	NoneType	None	# steps used as input
fcst_horizon	NoneType	None	# predicted steps
freq	str	D	datetime units. May contain a letters only or a combination of ints + letters: eg. “7D”
datetime_format	str	%Y-%m-%d	format used to convert “today”
datetime_col	NoneType	None	str data column containing the datetime
use_index	bool	False	bool flag to indicate if index should be used to get column

from datetime import timedelta

# Test
df_len = 100
n_values = 3
datetime_col = 'datetime'
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)
freq = "7D"
today = pd.Timestamp(get_today()).floor(freq)
df['datetime'] = pd.date_range(None, today, periods=df_len, freq=freq)
display(df)
max_dt = pd.Timestamp(df['datetime'].max()).floor(freq)
fcst_history = 30
fcst_horizon = 10
fcst_datetime = max_dt - timedelta(weeks=fcst_horizon)
print('fcst_datetime :', fcst_datetime)
start_datetime, end_datetime = get_fcst_bounds(df, fcst_datetime, datetime_col=datetime_col, fcst_history=fcst_history, fcst_horizon=fcst_horizon, freq=freq)
print('start_datetime:', start_datetime)
print('end_datetime  :', end_datetime)
dates = pd.date_range(start_datetime, end_datetime, freq=freq)
print(dates)
test_eq(len(dates), fcst_history + fcst_horizon)
test_eq(end_datetime, max_dt)

	value_0	value_1	value_2	datetime
0	0.0	0.0	0.0	2021-11-25
1	10.0	100.0	1000.0	2021-12-02
2	20.0	200.0	2000.0	2021-12-09
3	30.0	300.0	3000.0	2021-12-16
4	40.0	400.0	4000.0	2021-12-23
...	...	...	...	...
95	950.0	9500.0	95000.0	2023-09-21
96	960.0	9600.0	96000.0	2023-09-28
97	970.0	9700.0	97000.0	2023-10-05
98	980.0	9800.0	98000.0	2023-10-12
99	990.0	9900.0	99000.0	2023-10-19

100 rows × 4 columns

fcst_datetime : 2023-08-10 00:00:00
start_datetime: 2023-01-19 00:00:00
end_datetime  : 2023-10-19 00:00:00
DatetimeIndex(['2023-01-19', '2023-01-26', '2023-02-02', '2023-02-09',
               '2023-02-16', '2023-02-23', '2023-03-02', '2023-03-09',
               '2023-03-16', '2023-03-23', '2023-03-30', '2023-04-06',
               '2023-04-13', '2023-04-20', '2023-04-27', '2023-05-04',
               '2023-05-11', '2023-05-18', '2023-05-25', '2023-06-01',
               '2023-06-08', '2023-06-15', '2023-06-22', '2023-06-29',
               '2023-07-06', '2023-07-13', '2023-07-20', '2023-07-27',
               '2023-08-03', '2023-08-10', '2023-08-17', '2023-08-24',
               '2023-08-31', '2023-09-07', '2023-09-14', '2023-09-21',
               '2023-09-28', '2023-10-05', '2023-10-12', '2023-10-19'],
              dtype='datetime64[ns]', freq='7D')

source

filter_df_by_datetime

 filter_df_by_datetime (df, start_datetime=None, end_datetime=None,
                        datetime_col=None, use_index=False)

	Type	Default	Details
df			dataframe containing forecasting data
start_datetime	NoneType	None	lower datetime bound
end_datetime	NoneType	None	upper datetime bound
datetime_col	NoneType	None	str data column containing the datetime
use_index	bool	False	bool flag to indicate if index should be used to get column

# Test
df_len = 100
n_values = 3
datetime_col = 'datetime'
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)
freq = "7D"
df['datetime'] = pd.date_range(None, pd.Timestamp(get_today()).floor(freq), periods=df_len, freq=freq)
display(df)
max_dt = pd.Timestamp(df['datetime'].max()).floor(freq)
fcst_history = 30
fcst_horizon = 10
fcst_datetime = pd.date_range(end=fcst_datetime, periods=fcst_horizon + 1, freq=freq).floor(freq)[-1]
start_datetime, end_datetime = get_fcst_bounds(df, fcst_datetime, datetime_col=datetime_col, fcst_history=fcst_history, fcst_horizon=fcst_horizon, freq=freq)
test_eq(len(filter_df_by_datetime(df, start_datetime=start_datetime, end_datetime=end_datetime, datetime_col=datetime_col)), fcst_history + fcst_horizon)

	value_0	value_1	value_2	datetime
0	0.0	0.0	0.0	2021-11-25
1	10.0	100.0	1000.0	2021-12-02
2	20.0	200.0	2000.0	2021-12-09
3	30.0	300.0	3000.0	2021-12-16
4	40.0	400.0	4000.0	2021-12-23
...	...	...	...	...
95	950.0	9500.0	95000.0	2023-09-21
96	960.0	9600.0	96000.0	2023-09-28
97	970.0	9700.0	97000.0	2023-10-05
98	980.0	9800.0	98000.0	2023-10-12
99	990.0	9900.0	99000.0	2023-10-19

100 rows × 4 columns

source

get_fcst_data_from_df

 get_fcst_data_from_df (df, fcst_datetime, fcst_history=None,
                        fcst_horizon=None, freq='D',
                        datetime_format='%Y-%m-%d', datetime_col=None,
                        use_index=False)

Get forecasting data from a dataframe

	Type	Default	Details
df			dataframe containing forecasting data
fcst_datetime			datetime for which a fcst is created. Optionally tuple of datatimes if the fcst is created for a range of dates.
fcst_history	NoneType	None	# steps used as input
fcst_horizon	NoneType	None	# predicted steps
freq	str	D	datetime units. May contain a letters only or a combination of ints + letters: eg. “7D”
datetime_format	str	%Y-%m-%d	format used to convert “today”
datetime_col	NoneType	None	str data column containing the datetime
use_index	bool	False	bool flag to indicate if index should be used to get column

# Test
df_len = 100
n_values = 3
datetime_col = 'datetime'
df = pd.DataFrame()
for i in range(n_values):
    df[f"value_{i}"] = (np.arange(df_len) * 10**(i + 1)).astype(np.float32)
freq = "7D"
df['datetime'] = pd.date_range(None, pd.Timestamp(get_today()).floor(freq), periods=df_len, freq=freq)
display(df)
max_dt = pd.Timestamp(df['datetime'].max()).floor(freq)
fcst_history = 30
fcst_horizon = 10
fcst_datetime = pd.date_range(end=fcst_datetime, periods=fcst_horizon + 1, freq=freq).floor(freq)[-1]
test_eq(len(get_fcst_data_from_df(df, fcst_datetime, fcst_history=fcst_history, fcst_horizon=fcst_horizon, freq=freq, datetime_col=datetime_col)), 
                                  fcst_history + fcst_horizon)

	value_0	value_1	value_2	datetime
0	0.0	0.0	0.0	2021-11-25
1	10.0	100.0	1000.0	2021-12-02
2	20.0	200.0	2000.0	2021-12-09
3	30.0	300.0	3000.0	2021-12-16
4	40.0	400.0	4000.0	2021-12-23
...	...	...	...	...
95	950.0	9500.0	95000.0	2023-09-21
96	960.0	9600.0	96000.0	2023-09-28
97	970.0	9700.0	97000.0	2023-10-05
98	980.0	9800.0	98000.0	2023-10-12
99	990.0	9900.0	99000.0	2023-10-19

100 rows × 4 columns