Amazon Full Comment Analysis for Unusual Words (Misspellings, etc.)

NOTE: Please evaluate line by line. That is, do not evaluate the whole notebook all at once.

Import data

ExtractArchive["aggressive_dedup.json.gz"]

Clean/Parse Data

There will be error messages from this section. It is ok and expected.

UnixTime[{2009,1,1}] (*Only look at reviews from 2009 on*)

reviews=OpenRead["aggressive_dedup.json"]

data=OpenWrite["fulldata"]

line=Read[reviews,{String}];
i=1;
While[StringQ[line[[1]]],
split=StringSplit[line,"\": ",10];
t=split[[1]][[6]];
text=StringSplit[t,"\", \"",2][[1]];
u=split[[1]][[9]];
utc=ToExpression[StringSplit[u,", \"",2][[1]]];
If [IntegerQ[utc]&&utc>=1230768000,Write[data,{utc,text}]]; 
line=Read[reviews,{String}] ;
i=i+1;]

Close[data]

Close[reviews]

Streams[]

Scrub & Evaluate Data

fulldata=OpenRead["fulldata"]

Streams[]

ClearAll[i,j,misspell,date];
line=Read[fulldata];
j=1; i=1; date={};text={};
df2=Reap[While[IntegerQ[line[[1]]],
textwords={};
stringslist={};
misspell=0;
text=line[[2]];
(*CHECK FOR ENGLISH*)
If [RandomInteger[{1,100}]==1&&LanguageIdentify[text]==Entity["Language","English"], (*only takes 1/100 = 1% of entries and checks for English*)
text=StringReplace[text,{"1"->"","2"->"","3"->"","4"->"","5"->"","6"->"","7"->"","8"->"","9"->"","0"->"","$"->"","*"->" ","-"->" ","+"->" "}]; (*Remove numbers etc*)
text=StringReplace[text,{"'d"->"","'s"->"","&quot;"->"","&apos;d"->"","&apos;s"->"","n&apos;t"->"n't"}];
text=StringReplace[text,{"&gt;"->" ","&apos;"->"","/"->" ","\\t"->" ","\\"->" "}];
text=StringReplace[text,"\\"->" "(*,"."->" "*)];(*Remove paragraph breaks, contractions, etc.*)
textwords=TextWords[text];
For [j=1,j<=Length[textwords],j++, (*j is the particular word number*)
wordstring=textwords[[j]];(*discounts words 3 characters or less to avoid Roman numerals and common words like ok*)
characters=Characters[wordstring];
If [Length[characters]>3&&Not[SubsetQ[characters,{"#"}]]&&Not[SubsetQ[characters,{"&"}]]&&Not[SubsetQ[characters,{"."}]]&&Not[SubsetQ[characters,{";"}]]&&Not[SubsetQ[characters,{":"}]]&&Not[wordstring=="awol"]&&Not[wordstring=="AWOL"]&&Not[wordstring=="dvds"]&&Not[wordstring=="DVDs"]&&Not[wordstring=="iPhone"]&&Not[wordstring=="iPhones"]&&Not[wordstring=="iphones"]&&Not[wordstring=="iPad"]&&Not[wordstring=="iPads"]&&Not[wordstring=="iphone"]&&Not[wordstring=="ipad"]&&Not[wordstring=="ipads"]&&Not[wordstring=="apps"]&&Not[wordstring=="Apps"],(*checks for links and some slang and strange accent type characters and most emojis*)
If[DictionaryLookup[wordstring,IgnoreCase->True]=={},
misspell=misspell+1];];
];
];
date=DateValue[FromUnixTime[line[[1]],TimeZone->0],{"Year","Month","Day"}];
(*time=FromUnixTime[ToExpression[StringDrop[StringSplit[line[[1]],",",2][[1]],1]]];*)
If [Length[textwords]≠0,Sow[{date,N[misspell/Length[textwords]]}]];
line=Read[fulldata];
i=i+1;
];
];

Wrap Up

Close[fulldata]

Streams[]

Length[df2]

df3=Flatten[Drop[df2,1],2];

Length[df3]

Export["fulldf3_20C.dat",df3]

Beep[]

Clear[df2]

Can start here if evaluating IMPORTED data; otherwise, skip this section.

df1=Import["fulldf3_20B.dat","Table"]

Length[df1]

Clear[z]

df2A=Reap[For[z=1,z≤Length[df1],z++,
Sow[{ToExpression[StringJoin[df1[[z]][[1;;3]]]],df1[[z]][[4]]}]]];

df3=Flatten[Drop[df2A,1],2]

ClearAll[df1,df2,df2A]

Group & Compute Average Data

groupsAvg=GroupBy[df3,First->Last,Mean];

(*groupsCnt=GroupBy[df3,First->Last,Length];*)

(*groupsVar=GroupBy[df3,First->Last,Variance];*)

ClearAll[df3]

Display Data

mercuryretrogrades={{2009,1,11},{2009,2,1},{2009,5,7},{2009,5,31},{2009,9,7},{2009,9,29},{2009,12,26},{2010,1,15},{2010,4,18},{2010,5,11},{2010,8,20},{2010,9,12},{2010,12,10},{2010,12,30},{2011,3,30},{2011,4,23},{2011,8,3},{2011,8,26},{2011,11,24},{2012,11,7},{2011,12,14},{2012,3,12},{2012,4,4},{2012,7,15},{2012,8,8},{2012,11,6},{2012,11,26},{2013,2,23},{2013,3,17},{2013,6,26},{2013,7,20},{2013,10,21},{2013,11,10},{2014,2,6},{2014,2,28},{2014,6,7},{2014,7,1},{2014,10,4},{2014,10,25}};(* UK time*)

tsAvg=TimeSeries[groupsAvg]

(*tsCnt=TimeSeries[groupsCnt]*)

(*tsVar=TimeSeries[groupsVar]*)

ClearAll[groupsAvg,groupsCnt,groupsVar]

Averages

tsA=TimeSeriesWindow[tsAvg,{{2009,1,1},{2014,7,23}}]

res5=MeanFilter[tsA,(*20000000*)20000000]

DateListPlot[{tsA,res5}]

Show[%,ImageSize->Full]

diffA=TimeSeriesThread[First[#]-Last[#]&,{TimeSeriesWindow[tsA,{{2009,7,1},{2013,7,1}}],TimeSeriesWindow[res5,{{2009,7,1},{2013,7,1}}]}]
DateListPlot[diffA,{2009,9,1},GridLines->{mercuryretrogrades,None}]

dA=diffA[[2]][[1]][[1]];

ListLinePlot[Abs[Fourier[dA]], PlotRange->All]

ListLinePlot[Abs[Fourier[dA]][[1;;20]]]

data=dA;
f=Abs[Fourier[data]];
n=Length[data];
Clear[findPeriod];
findPeriod[data_,threshold_]:=Module[{fs,s1,s={},i,a0f,af,pf,pos,fr,frpos,fdata,fdatac,n,per},n=Length[data];
fs=Fourier[data];
s1=Drop[fs,-Floor[Length[fs]/2]];
For[i=1,i<Length[s1],i++,If[Abs[fs][[i+1]]>threshold,AppendTo[s,i+1]]];
a0f=Abs[fs[[1]]]/Sqrt[n];
af=2/Sqrt[n] Abs[fs][[s]];
pf=Arg[fs][[s]];
{a0f,Transpose[{s,af,pf}]}]

Clear[reconstruct];
reconstruct[data_,fp_]:=Module[{n},n=Length[data];
Show[ListLinePlot[data,PlotStyle->Black],Plot[fp[[1]]+Sum[fp[[2,j,2]] Cos[2 Pi (fp[[2,j,1]]-1)/n t-fp[[2,j,3]]],{j,1,Length[fp[[2]]]}],{t,0,n},PlotStyle->Red]]]

TableForm[Sort[findPeriod[data,0.002][[2]],#1[[2]]>#2[[2]]&],TableHeadings->{None,{"Freq","Amp","Phase"}}]

Mean[data]

N[n/13]

pos1=9;
pos2=13;
per1 = N[n/(pos1-2+2*((n-pos1-1)/n))];
per2 = N[n/(pos2-2+2*((n-pos2-1)/n))];
pdata=Table[N[Mean[data] +0.00014768346951039754`*Sin[(2 π /per1)(x +2.903383954382567`*(per1/2 π ))]+0.0001356465804712962`*Sin[(2 π /per2)(x -1.0587255021189352`*(per2/2 π ))]], {x, n}];
DateListPlot[pdata,{2009,7,1},PlotRange->{-0.0005,0.0005},GridLines->{mercuryretrogrades,None}(*,Filling->Axis*)]

Show[%,ImageSize->Full]

Max[pdata]/Mean[tsA]