
if load_data==1

% *********************************************************************
% load the GSW yield dataset
% *********************************************************************
[num,txt,raw]=xlsread('data_all.xls','GSW_data');
indtmp=isnan(num(:,1))==0;
num=num(indtmp,:);
tmpdates=datevec(txt(2:end,1));
tmpdates=tmpdates(indtmp,:);

% back out yields using the parameters 
BETA0=num(:,1);
BETA1=num(:,2);
BETA2=num(:,3);
BETA3=num(:,4);
TAU1=num(:,5);
TAU2=num(:,6);

matur=12:120; 
yc= nan(size(tmpdates,1),length(matur));
for t=1:size(tmpdates,1)
        nTau1          = (matur/12)./TAU1(t);
        nTau2          = (matur/12)./TAU2(t);
        yc(t,:)	= BETA0(t) + ...
                                BETA1(t).*((1-exp(-nTau1))./nTau1) + ...
                            	BETA2(t).*((1-exp(-nTau1))./nTau1 - exp(-nTau1)) + ...
                            	BETA3(t).*((1-exp(-nTau2))./nTau2 - exp(-nTau2));
end
yc=[tmpdates(:,1:3),yc];

data_yield=nan(T_m,size(yc,2)-3);
for i=1:T_m
    indtmp=(((yc(:,1)==calds_m(i,1))+(yc(:,2)==calds_m(i,2)))==2);
    tmp1=yc(indtmp==1,:);
    if isempty(tmp1)==0
        data_yield(i,:)=tmp1(end,4:end); % end-of-month observations
    end
end

% construct a name vector 
for is=1:length(matur)
    tmp = ['YIELD' num2str(matur(is))];
    names_yield(is)=cellstr(tmp);
end

% vector of transformation codes 
tcode_yield=1*ones(1,size(data_yield,2));

% convert to quarterly 
if m2q==1
    data_yield = mtoq(data_yield,calds_m,calds_q,0);
end


% *********************************************************************
% load the MOVE index
% *********************************************************************
[num,txt,raw]=xlsread('data_all.xls','Daily_MOVE');

tmpdate =num(:,1:3);
tmpdata=num(:,end);
name_move=raw(1,4);

data_move=nan(T_m,length(name_move));
for i=1:T_m
    indtmp=(((tmpdate(:,1)==calds_m(i,1))+(tmpdate(:,2)==calds_m(i,2)))==2);
    tmp1=[tmpdate(indtmp==1,:),tmpdata(indtmp==1,:)];
    if isempty(tmp1)==0
        data_move(i,:)=tmp1(end,end);
    end
end

% transformation code
tcode_move=1;

% convert to quarterly 
if m2q==1
    data_move = mtoq(data_move,calds_m,calds_q,0);
end

% standardize the MOVE Index
tmp=data_move;
tmp1 = tmp(isnan(tmp)==0);
tmp2 = (tmp1-mean(tmp1))/std(tmp1);
data_move(isnan(tmp)==0) = tmp2; 


% *********************************************************************
% calculate realized S&P500 volatility
% *********************************************************************
[num,txt,raw]=xlsread('data_all.xls','Daily_SP500');

sp500_d=num(:,end);
tmpdate=num(:,1:3);

RVol=nan(T_m,1);
for i=1:T_m
    indtmp=(((tmpdate(:,1)==calds_m(i,1))+(tmpdate(:,2)==calds_m(i,2)))==2);
    tmp1=[tmpdate(indtmp==1,:),sp500_d(indtmp==1,:)];
    if isempty(tmp1)==0
        tmp2=tmp1(:,end);
        tmp3=100*(log(tmp2(2:end))-log(tmp2(1:end-1)));
        RVol(i)=12*sum(tmp3.^2); 
    end
end

RVol=sqrt(RVol);

name_rvol={'RVol'};
tcode_rvol=1;
data_rvol=RVol;

% convert to quarterly 
if m2q==1
    data_rvol= mtoq(data_rvol,calds_m,calds_q,0);
end


% *********************************************************************
% load the ACM expected short rates and term premiums, 
% average 3-month TB survey forecast, GZ excess bond premium, 
% Philadelphia Fed leading indicators, and the LMN financial uncertainty index
% *********************************************************************
[num,txt,raw]=xlsread('data_all.xls','Other_monthly_variables');

name_other_monthly_var=txt(1,2:end);
tcode_other_monthly_var=num(1,:);

tmpdata=num(2:end,:);
tmpdates= datevec(raw(3:end,1)); 
tmpdates=tmpdates(:,1:2);

data_other_monthly_var=nan(T_m,length(name_other_monthly_var));
for i=1:T_m
    indtmp=(((tmpdates(:,1)==calds_m(i,1))+(tmpdates(:,2)==calds_m(i,2)))==2);
    tmp1=tmpdata(indtmp==1,:);
    if isempty(tmp1)==0
        data_other_monthly_var(i,:)=tmp1;
    end
end

% convert to quarterly 
if m2q==1
    data_other_monthly_var = mtoq(data_other_monthly_var,calds_m,calds_q,0);
end


name_other_monthly_var(ismember(name_other_monthly_var,'ebp'))={'GZ - excess bond premium'};

tmpind=ismember(name_other_monthly_var,'Financial uncertainty h=1');
name_other_monthly_var(tmpind)={'LMN - financial uncertainty'};

% Standardize the LMN financial uncertainty index
tmp=data_other_monthly_var(:,tmpind);
tmp1 = tmp(isnan(tmp)==0);
tmp2 = (tmp1-mean(tmp1))/std(tmp1);
data_other_monthly_var(isnan(tmp)==0,tmpind) = tmp2; 


% *********************************************************************
% load the FRED dataset
% *********************************************************************
if m2q==0
    
    [num,txt,raw]=xlsread('data_all.xls','FRED-MD');
    tcode_fred=num(1,:);
    tmpdate = datevec(raw(3:end,1));
    tmpdate=tmpdate(:,1:2);
    tmpdata=num(2:end,:);
    name_fred=raw(1,2:end);
    
     % apply some transformation
    data_fred=prepare_missing(tmpdata,tcode_fred);

    istart=find((tmpdate(:,1)==firstdate_m(1))+(tmpdate(:,2)==firstdate_m(2))==2);
    iend=find((tmpdate(:,1)==lastdate_m(1))+(tmpdate(:,2)==lastdate_m(2))==2);    
    data_fred=data_fred(istart:iend,:);
    data_fred_raw=tmpdata(istart:iend,:);
else
    %- Adjusted TFP
    [num,txt,raw]=xlsread('data_all.xls','Adjusted-TFP');
    name_adjust_tfp=txt(2,2);
    tcode_adjust_tfp=5; % set this for irfs and fevds, but note the series is already transformed
    data_adjust_tfp=num(62:289)/4;
    
    %- FRED-QD
    [num,txt,raw]=xlsread('data_all.xls','FRED-QD');
    
    inclcode_fred_qd=num(1,:);
    
    tcode_fred=num(2,:);
    name_fred=raw(1,2:end);
    tmpdata=num(3:end,:);
    
    % apply some transformation
    data_fred=prepare_missing(tmpdata,tcode_fred);
    data_fred=data_fred(15:242,:);
    data_fred_raw=tmpdata(15:242,:);            
end

multipby100=ones(1,size(data_fred,2));
for i=1:size(data_fred,2)
    tc=tcode_fred(i);
    
    if tc==4 || tc==5 || tc==6
        multipby100(i)=100;
    end
end
data_fred=data_fred.*(ones(size(data_fred,1),1)*multipby100);


% *********************************************************************
% load internationl bond markets data
% *********************************************************************
%-- Japan
[num,txt,raw]=xlsread('data_all.xls','Japan');

tmpdate=num(:,1:3);
tmpdata=num(:,4:end);
namesvec=txt(1,4:end);
namesvec=upper(namesvec);
namesvec=strtrim(namesvec);

data_m=nan(T_m,length(namesvec));
for i=1:T_m
    indtmp=(((tmpdate(:,1)==calds_m(i,1))+(tmpdate(:,2)==calds_m(i,2)))==2);
    tmp1=[tmpdate(indtmp==1,:),tmpdata(indtmp==1,:)];
    if isempty(tmp1)==0
        data_m(i,:)=tmp1(end,end-length(namesvec)+1:end);
    end
end

names_yield_ja=namesvec;
data_yield_ja=data_m;


%-- UK
[num,txt,raw]=xlsread('data_all.xls','UK');

tmpdate=num(:,1:3);
tmpdata=num(:,4:end);
namesvec=txt(1,4:end);
namesvec=upper(namesvec);
namesvec=strtrim(namesvec);

data_m=nan(T_m,length(namesvec));
for i=1:T_m
    indtmp=(((tmpdate(:,1)==calds_m(i,1))+(tmpdate(:,2)==calds_m(i,2)))==2);
    tmp1=[tmpdate(indtmp==1,:),tmpdata(indtmp==1,:)];
    if isempty(tmp1)==0
        data_m(i,:)=tmp1(end,end-length(namesvec)+1:end);
    end
end

names_yield_uk=namesvec;
data_yield_uk=data_m;


%-- Germany
[num,txt,raw]=xlsread('data_all.xls','Germany');

tmpdate=num(:,1:3);
tmpdata=num(:,4:end);
namesvec=txt(1,4:end);
namesvec=upper(namesvec);
namesvec=strtrim(namesvec);

data_m=nan(T_m,length(namesvec));
for i=1:T_m
    indtmp=(((tmpdate(:,1)==calds_m(i,1))+(tmpdate(:,2)==calds_m(i,2)))==2);
    tmp1=[tmpdate(indtmp==1,:),tmpdata(indtmp==1,:)];
    if isempty(tmp1)==0
        data_m(i,:)=tmp1(end,end-length(namesvec)+1:end);   
    end
end

names_yield_de=namesvec;
data_yield_de=data_m;


% 10-year yields and components
str_vec={'ZCY_120','ZCYHAT_120','ZCYER_120','ZCYTP_120'};
ind_ja=[]; ind_uk=[]; ind_de=[];
for i=1:length(str_vec)
    ind_ja=[ind_ja,find(ismember(names_yield_ja,str_vec(i)))];
    ind_uk=[ind_uk,find(ismember(names_yield_uk,str_vec(i)))];
    ind_de=[ind_de,find(ismember(names_yield_de,str_vec(i)))];
end

data_yield_ja=data_yield_ja(:,ind_ja);
data_yield_uk=data_yield_uk(:,ind_uk);
data_yield_de=data_yield_de(:,ind_de);

names_yield_ja={'ZCY_120_JA','ZCYHAT_120_JA','ZCYER_120_JA','ZCYTP_120_JA'};
names_yield_uk={'ZCY_120_UK','ZCYHAT_120_UK','ZCYER_120_UK','ZCYTP_120_UK'};
names_yield_de={'ZCY_120_DE','ZCYHAT_120_DE','ZCYER_120_DE','ZCYTP_120_DE'};


data_yield_ja_uk_de=[data_yield_ja,...
    data_yield_uk,...
    data_yield_de];

name_yield_ja_uk_de=[names_yield_ja,...
    names_yield_uk,...
    names_yield_de];

tcode_yield_ja_uk_de=ones(1,length(name_yield_ja_uk_de));

% convert to quarterly 
if m2q==1
    data_yield_ja_uk_de = mtoq(data_yield_ja_uk_de,calds_m,calds_q,0);
end


%-------- combine datasets and save variables disregarding _m or _q
if m2q == 0
    data=[data_yield,data_other_monthly_var,data_move,data_rvol,data_fred,data_yield_ja_uk_de];
    data_raw=[data_yield,data_other_monthly_var,data_move,data_rvol,data_fred_raw,data_yield_ja_uk_de];
    name_vec=[names_yield,name_other_monthly_var,name_move,name_rvol,name_fred,name_yield_ja_uk_de];
    tcode_vec=[tcode_yield,tcode_other_monthly_var,tcode_move,tcode_rvol,tcode_fred,tcode_yield_ja_uk_de];
    
    
    %-- vector of codes indicating how variables are used in the factor estimation
    % 1 include in the yield curve factors estimation
    % 2 include in the macro factors estimation
    % 3 otherwise
    catcode_vec=[ones(1,size(data_yield,2)),...
        2*ones(1,size([data_other_monthly_var,data_move,data_rvol,data_fred],2)),...
        3*ones(1,size(data_yield_ja_uk_de,2))];     
    catcode_vec(ismember(name_vec,{'ACMTP02','ACMTP05','ACMTP10','ACMRNY02','ACMRNY05','ACMRNY10','TB3SAvg'}))=3;
    
    %-- vector of codes indicating whether variables are used in the factor estimation
    % 1 include in the macro or yield curve factors estimation
    % 0 otherwise
    inclcode_vec=zeros(1,length(tcode_vec));
    inclcode_vec(catcode_vec==1)=1;
    inclcode_vec(catcode_vec==2)=1;
    
    % number of observations
    T=T_m;
    
    % name of the data file saved for future
    str_tmp = 'data_m';
else
    data=[data_yield,data_other_monthly_var,data_move,data_rvol,data_fred,data_adjust_tfp,data_yield_ja_uk_de];
    data_raw=[data_yield,data_other_monthly_var,data_move,data_rvol,data_fred_raw,data_adjust_tfp,data_yield_ja_uk_de];
    name_vec=[names_yield,name_other_monthly_var,name_move,name_rvol,name_fred,name_adjust_tfp,name_yield_ja_uk_de];
    tcode_vec=[tcode_yield,tcode_other_monthly_var,tcode_move,tcode_rvol,tcode_fred,tcode_adjust_tfp,tcode_yield_ja_uk_de];
    
    %-- vector of codes indicating how variables are used in the factor estimation
    % 1 include in the yield curve factors estimation
    % 2 include in the macro factors estimation
    % 3 otherwise
    catcode_vec=[ones(1,size(data_yield,2)),...
        2*ones(1,size([data_other_monthly_var,data_move,data_rvol,data_fred,data_adjust_tfp],2)),...
        3*ones(1,size(data_yield_ja_uk_de,2))];
    catcode_vec(ismember(name_vec,{'ACMTP02','ACMTP05','ACMTP10','ACMRNY02','ACMRNY05','ACMRNY10','TB3SAvg'}))=3;    
    
    %-- vector of codes indicating whether variables are used in the factor estimation
    % 1 include in the macro or yield curve factors estimation
    % 0 otherwise
    inclcode_vec=[ones(1,size(data_yield,2)),...
        ones(1,size([data_other_monthly_var,data_move,data_rvol],2)),inclcode_fred_qd,ones(1,size(data_adjust_tfp,2)),...
        zeros(1,size(data_yield_ja_uk_de,2))];
    inclcode_vec(ismember(name_vec,{'ACMTP02','ACMTP05','ACMTP10','ACMRNY02','ACMRNY05','ACMRNY10','TB3SAvg'}))=0;        

    % number of observations    
    T=T_q;  
    
    % name of the data file saved for future    
    str_tmp = 'data_q';
end

save(str_tmp,'T','data','name_vec','tcode_vec','catcode_vec','data_raw','inclcode_vec');

else

if m2q==1
	str_tmp = 'data_q';
else
    str_tmp = 'data_m';
end
load(str_tmp);
    
end