From e2b1f320cf8239cb9872f5176cff0f0b380f6f51 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 6 Dec 2017 11:57:17 +0100 Subject: [PATCH 01/36] Add data.world to dialect selector --- app/constants/constants.js | 6 ++++-- app/images/dataworld-logo.png | Bin 0 -> 13251 bytes 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 app/images/dataworld-logo.png diff --git a/app/constants/constants.js b/app/constants/constants.js index df4ac8079..b12a3ec23 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -14,7 +14,8 @@ export const DIALECTS = { IBM_DB2: 'ibm db2', APACHE_SPARK: 'apache spark', APACHE_IMPALA: 'apache impala', - APACHE_DRILL: 'apache drill' + APACHE_DRILL: 'apache drill', + DATA_WORLD: 'data.world' }; export const SQL_DIALECTS_USING_EDITOR = [ @@ -194,7 +195,8 @@ export const LOGOS = { [DIALECTS.MSSQL]: 'images/mssql-logo.png', [DIALECTS.SQLITE]: 'images/sqlite-logo.png', [DIALECTS.S3]: 'images/s3-logo.png', - [DIALECTS.APACHE_DRILL]: 'images/apache_drill-logo.png' + [DIALECTS.APACHE_DRILL]: 'images/apache_drill-logo.png', + [DIALECTS.DATA_WORLD]: 'images/dataworld-logo.png' }; export function PREVIEW_QUERY (dialect, table, database = '') { diff --git a/app/images/dataworld-logo.png b/app/images/dataworld-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..3ca8803846dc9323afce00b6291615b4b4bcb8fd GIT binary patch literal 13251 zcmY*=Wl&sA6E3pIqJbbG*aC|McL?qa1Shz=ySr9--ET+&qrCm<>ttnPyRy9x={jNN9UWz1>l?*Lv0+I|C4hgDy2@3; zkuueClwC+1j<0007(BzI8#%wc?B}hhy0ylsLkW4*^M)oD%set>(Cur6Ecb zVUK7HTY2f&l8M7fNqfC5)_+r0G9d&9m)TJ1c0@DSU@+L8IPGaj#RMxlwVjl7-?;!l#NNt@^`T`dm*4luThNn%2O5U3XYdQ zn{4LIrB_uv?CvMK3AjU58LPMCz)vnl&SFm^hnay*r#s%i5#8ZI)z5>Ks(jCY@{qm`p zB_<96rXKx~O&~QSK1#*Mwnqf%-ly6ceRwo}Zp5s`*$ARB)tMRjs=t;8dVSc$8FT zKahm>7ap}pNPlYsnlLmvSxw~s#S|-h?)kYok;APllwVrlrUGYyfbzKGSx5`gL<-hM z@31dGS9`@Otu5(aoW_$K$$<}nYMAQ38X#9ERjqluoZrko-mb^iZEQ{fMtm2wAd$Zy z3ok4go}pBgW(#MrTu6PN4;VfjKr~O0 z_{T0~dTwo}loWSf22>#w@SXc-{zsv-)8FL7O@f*qA4-=Hiz;8YMymVtR$Pf&2Ak{j zP@59gr+BXoXk4Ey3kfn>HzKyoTl<*E+Xi2Iw@0*?={Jr=dm_YqJY@$A0ruH#(>`7Y z3}w8|doo&;IuU*>0$bR1FBAzO%~^EC7sx+hFu$6Gy8Vc*GE1d|s+y)9UH~fmf;};W zY6yI3qJye6Q-J9*$|lwicSAHQaBs((118IbnBEZ=B}gIYoExJ|BIp@`xwCgy$7RP) zk{c5{L@s7R`4Sl4mA) z2^hT_?!JJeomj?@WxoV~19rXjr~QMkEU2`S)&>`Q_ZAQ6|KXik&D2-fbr2-!>~)&= zC9)fxE}DYB3EOdLU^9M@^jHD&e^R?s4B){)8X|JhBj-2eS0MkMEw3iv zNh+5vX@Ra^=qCas$BQkDJOg?#qa~s)VcGC!zG)1{)Eza5I(*gNx2Mk`Br0f5*3&aQ zJ0;G_m>9 z*F*K?nor$eY@5wqHJhg1>z9d=;-oWS!&ZNEaIHo|4e|H=<0JNNgvdfz<{9EytQ+z) zBDHxXjRiAHQu0#xFyU_ms+o&_7kATtO%3_;;^8>3s%JIE>o!Lg#6!CTLYj4HK$+iW ztIghrlxmT|x51)02VU#1cImI&hN|90(R+^i(igvj8RmcOO|RRfvTAWdYv^8?n#W)52d@m5zqe_!u_ zb>6xaE$Q9-bfeqG#zW5PHYkLtN!nS({DgD{3+|trBZ4ZD49J9L83Cz{Jt*R-W8{w&r^fX({=Clp{x~o1*-t1# zH6U!C1{4+)NUr;4Pahz!oohSgkDfI~+=C6FleO)U1V+~dt84mkyvd|jq0swyEv)Ha zj^Go@0Oa4CSiEr4?0X6(-p-%xFa{ncaK510U9_{_LO>NXT@L~{d3erC*hUsT^?~)O z^gh08A?N2_O?Ogr+qAF>k3*;UaLqY?&4dlwe{uMZ0Knf?2a{#rtx6An3e{>Ok#9EZEUmP z6wQ5F1X%ko2s&N#X3tL|cz-q3%U`dozXFIWr;ehNxwOahTya=(kn&pBniGV`$q8O_ zfyr(qIur|^chr#bLi#(SyHX7*nGzKnU(U*y?EX-xqYJxwbyGSSc1J30-a4lMqlktn$XSAN%lt# zo|li@fQCL8)oPVK6JDpoBAq0V&o$`lj#~G_4FMD0)~z%Z%^pH0;&E6*)0hN6L&QPUhQcuh-fze-FTP+F=fi%SPu6b8gV<+C(dK zv0HcCp=g?@S|KINdmfL`vz;amHOn0rL};e2SUXVql`;Qwq{|xWx^?fWz%!`BHhb=;V^9D` zI{haY;4?oGJf;vzOjq}8Cib_rZ%ohbt$1rPcYic=cu{3#CTQG#^lb~(X{K}u)RupZ zV6=!Ax-_TdxETlg2&?NQPyhJt;uH1R%`7cTW~$crsfqdDt34lVGyu(a^U56K$eh1@ z{)t`yxieJ=S=^bSN5%Gxm?Yb7gXwxsM)dCH8yol+^A9s*-*2Izl~_P9CN7>#vyA~? z)4}%d8;9WGZ*8+ljOczT>hs+Wsa>S!NI9WoUontK9rT%q-lB zpgC%EUC;DT2fB#q{!x$w6ix!qn202iV;mG0Vq#t$G%GbXgI&|)Xb~hzw4VCGP(DQx zoY|I|flE&?_dXw5=MKr|}(r``D)w+X#X-3`u@ z3|hAQ)LcX$;gAWip4KxePRO8yFIC1j@fM7kLZ%?YFOFK^@(e*^f~u~EQw>5g0b3+>?kvm{_~MgqCWAfgR5sjFfZ8gR@JVl?oc95*`;pbvvOt0 zxn3n*NZ-(R7e@>lgQ7ne+6XEwfOs=0Z40*fz#h+0psR_;7&gnig#NWG6N9oUH zn;YJ%pT(`erJkw@{=tB5wtF1oh=&=?UgK)sM%f$^?{p!^@$l1x;fLA+!4saNfY{7l z&EMHl>@yxF|0chh@g;o(YPKRj?SBt8=|Z#+cxy}C2@Mq0dwH>LhLGuhGFW;rW0ia6#1@Tx!M6aOwsD~VNMy4hyH=&PsnHhk<}HV<({sj@TM~~Pe z3;F;tu|SKkHbgD#Dc$z{#XmC z?YI=RZta@AzvHLSCC*d>8u|~f>WN5vj5$OSYabC1Xw1mO`wUaejU}fI5}lL~6+HB4&oa)? zx&)#48fbaX6gR_AeRuHFOa%B_li-re9GrgLIl1;zBh>7ciZm`2Tg%Ue78{~egG0<& zAU1XXo;B^0?pG?mD~zlj!+H-Iq#+IBzq zy;|gUN|VEJfZ#*g06f;SO63Eg$2kK9{VU9C%q9Pj^aNX%}CUN8#8V=R6Z`p zE>qs0a87+yV4(^Mx78Fb32ZLl|IXj))s#UJ+~PRIFTH{|xWO)xlp9qTVpok7(L;Oe zc~@c_|0yyjr&j9jLUSd4>GBb=x=#-+?@hKAg0n8D9Q^tpp0hb#^jS=m8&=~BRn8$j zzY^o#;{8{s#EE&YtRTaGJscB>TLcP-X89yNcIx3?7aeOC4}5ItHsbFds)GM1`T)uu=-50XiX5^# zm>qX!`cHYjO5N9MV_a`y1|zd()c+hEln|%jS#*Kj8rHbZN+u5ghv-6_*&6K>P%7P* zr|Bc9{+k)HigZniP9r@flB;1t>0wG}J{l%^QIT0D1`p{6cV_puII9I)4o=QCEvg6x zi_nt>By9O*FY3OY0B}KTtK-$%`wa^fmm_tIZ-XFXD$6L^$mj>t_oqLEnNd0AV-=58 zL*5flsn*kQHCst(D)5?aEhJKP&rE8K;pRHy`kSn0A*blCk-_Pg^S5EG#xv&B&Xv6n zw5~#5L2Ixjd2LQRg%x{y9DRMbP6rH@*4C!MAL7D~7t-sS4G2)$l#KR8e{seT3ijkDl4_h9R23SW77Dgl0MDKe$Uwz{QTsn;c|0DAWZI)JU(vQW?-B& z+=;tQq5P|?yxizA^He8Ge{ACd(HJ`E9vkta&v0yvPHsejj)O`7_$8W$LdNT28&?7f zDY@`$|mkES}=BkHw9klVZ$ltaVwR!qO~E+9QU2N+aTk}*W`6Oa_7dWYS=5vpZBvA{|w z&%SS?8V?%MZI?wjKw_k7yo+4kYZU!lgW6?@*wp;+mDS5%3Iqf4Df@CB3SbC$et9`m zUVhuLS6s8+v+X2yOcYF%P>SL9wa;43>F>Ylx?Yo1G4eP-AV*j>3btgSh@O%r<++y{ zgvh`OAAKqF1hSWQNv`TT;d|gi+t9jPj{p# z-q-DJ6pJv)lR~n5c~~-&M`09|P;Q`l-mIpRot~f!qA)oR(fE`RHJRSEY@GY9+e;_| zJ%%?|n1}%tZWSUQ-}LXiL&}fQP6v$r-3-74f4}99H|GDu`2Aj$ib%(usHC)6OJ@7q zyfB-!8LJlD?k8CiJg#4!FjG~$?>&1k2lDd_RhKGs8i5VqbU+5#fP98&eZQ+4NvxZ_ zJ=6gi`*5mfK>BB#-^U_*AZ;Nqp-?_OD_Xhtjct~uKcQxVy|Q=z#JX+rF)l*l`*(id z{(QC^jvYN2X}W57G{nHQwKnA}{@vb1EUy>6&^E|FcH9Qb!2H%RxB4y|Z!a#Ah>#Yn5`S;2*hcgE$Dx0{2 z%XvCHEt+;wx0_PjL|H6!OsXV54U1({rGwz!2H?J{_HhFuoZr?(L#F@bk?XXq(^Nx> zVg2q$$r>Tm!sIXT*d>jPdU?d5%24~b-w2-9#os;SNci2t8|%}5>b#5m`V#x@vW0%< zSMOf#0?i!AC#p={X%x9S%PJo7N=Qy!If^YZK5*o+^&#p%2St7y(fsO`0cOYsVa+ zM62gb8Wwo^gKn(bQJu9xmmUR%O16MTO7OMc{SQsv@Yz*1(2-6&0h?28V&FD9gl2Cq zSzm25+&wPb2#%BUUQN4Gx5F1T0q^9mLN!VpW1Ww&VU62)K;2?R`|MQ?kLNXD=L1;D zdvEdNBycuWV>UhdGuz?LfPBrWET{MGeedI1ug51|+sHLrGeATl#LElm z3~N(4-dZ_qYH_g}jKn0;P5y&8nkjXVh}$0AOk>gEB+cirjYbh4A*S6nA5Nr*IA0J< z5g=_N@9!p`=hUx((CAuT8_yrCgxwblHA>JogY zq4`Yepvy<#@pP+0h17)2`(|vnmBqR4vqXp-^+vQH|F|JjCaHY!w}FhZQIy$akMVZijM{De+NX7E8o@;ahsV_WChbZN zD^5^nbODf%2kjXVcSEb;?4M`bkNXg@ZoI;SOkN@bKkf>qOyni>`rN-B$EHdb+kGPx zlJe!6ZMJizF5y#ch2>LB)bV|LE9=*#C;D8DaVHb|vq9=rEYJIzMIl%xs1e}56|o*g zk90BD6mlVh=cMcNCS!f-TFI?zHZ5s-aT3vMs@@Pte#c@9N(X1s`IDgU+S&`SYS5{w zh3q>hvIVcmoU8@tfGI2rMP~60`{t*WCbDSF2{d*UxcUNdUmH!eI><`5t*$(kC%iTp zrX+XqXUoTbPa=ebVO|}X3p34_B?T!ZID}Y_wX4DI(N%^_X zv6kF5@vj?^q2anLoC8gHt~XYyQj_YHoU3On1-(mq4+RU@%-${LSgv3=ljrtSdz00=VUpmv45I@2a-_% zgtT#(Lz!WucnsNPIV4?Ba%d1B02yZ4(iq<=K@lhC;IX~4<^5y}Bqesdj%izv_~Las z8ge8KZ|q!qt3hh0q@;o~8uT}8e!;plJe!PcZ_|_FS_Be2UrPE~p*x>22ekFoJ~gln zWF&`x?eNbNPc4#11Al#G+9gDT=bsZRbg+{1R?UWoT|AcD*;wbCIk)N}U)e#pzx%jn z6r(hNeT=f*<0Kv|8Cr*pvmq@8XW%3s9tX=;1P^R^`yk_sq`WG0tH@%UAI8aUjY`vd}GrobaoQoCieB<%b4wa0`?H%H7h72OBx5 zD4+P{-VS<(ccPUD2ZSZJ$Q(qJx!I9Z(2VNO`-t%#7vpW~%=(l-g0$ggoZa8Kas`z9 zB8ppUdhCOp@0iQX(Z{X7ieqyiVjwJfgKRB&SaCwi)I4NDah#>#1PPEZ`?aa%uw0W) z^SpufZOAxMWNz2*f>?KFIDeI1_0RJasG?X~T>d!n6i~RQYP+=L$@HK6P~?EWY(MdK zBBO@zfB@{S=}wPs?0AXOD_!H}kYC{nYE1P6w?@Dt|J{`yvG?i>+#PBZji{ucFI>t5 zhjRf?!dj$k+Jh1VX(`6{wZvXCNz@^_}KTWn%jl# zFz&)ah>{v6IFnASV4a0Nn!0fkA_14jt#}QN{DXy^wXvjDGp|43SxFMpy|Q6({8Csf z(`eb?Y^={gv-bO1F9f!`r?~q>iz~;Kt>~zE2a~(OcGWm8kOpEIRr6kQTD`LAhK7}F*r%Hy0aOyy zx14bK6aa$ikw3f1I@oz`)uvVI-}UBKQDHC24#U6zd+X`n9`$SCwcXWy@2{nPNl&A% z9wh$3S(G+!Gf}07ca{X?+v2vzfmNqMfJVa98OF1fXeAWOnpxNqt1hjq_8#q4FkrnF z1>woN_m`7hUI%57BPxmBZ=>}mn=9=6W@Y4Lh2kf%cKu8X@eSR|fZ&+x(!yVN$Lfq_ zz$MMT;1Mlum+3YIoOU9JL(^I~8MfPwZdUz$F+8?8`U5PsStblMdG;D>3!1Nh2tHPA zBX`cEP}_zTZ2}87jnMgp#pBm~&7B{9io}RXj)>qD{qHhfyzENrK5$J6Fw_H{pb5j| zm(e4)uw>Lb_42$N~? zd5kS9U59GB3}nm5!{J_La@-<;MIYh_4-S?vS{r61c$H5{V8l@`Dix;NFX)lpP3*F{ zJ^Mnf^RIG4=8v``eEv8BlZ%k2CJ0o}-MOAf@iBK5s*AuFA zHy{SqvwaN+4vP8kR$y`EVrR$qxd9`$j2sp8eP^LFGzhtbB1-+s%!hUK?}Tx1*dnvA zKtR_TXJUM}wius#X^gX_{_0&5p4crKeEV5| z0hvsx0KGt*o1D0sI6Z>e6e1@tvE6CgQeav zJDyug@CiYnFErER=$!J_P4rO}Ami;HJUM8Ven8u+!J{u;+T6s3u5c;~{%gfCP)bsl zo2b%DBa-IjuQPVyyL9od8X`1xX*eNFKAznG@3`aD7e#68e}S2=5`!AzG#H1=5($c; zFrHs5Ma0nk3?g{^@R>GutL+*hx$(o4 ziUch*_oDYFhEa)9vPpXTWdUkry%|9K#z?N()L3kx&i)_wd$3tg6Jg@N-@X7Bn_s zzU55R09l85#Vp+`^D=?3#uskrWC$$(*K1qkTiFj5WxCD9BtAflCVe!R0Q;kV49=p_B53A!XOklz<0;osJ6yt0p+MkPpVa22!BP1jdkW_Hi(ZdYF< zq;1y6tWiG34-^6S2kNn5)^O_sDD{d`aNywpz_}_6^g})?qo4=E{NaH1vB*txpl1}{ zYnO*Ea#DDZCNhX1&i*T;(qknFW>`J-%hD%0SPLKO3=2U7ZQ}jJ`Eoi)0^yW`IQCChQ{4Yrwn0F|Z zW19#{pC?+B)sXILXz*+c%7`WG=dLjG3n}ah9)HIhj1IDuf373)oWq9x1_7gy(u9*F zy1sI)&r+&2_DF>S5)RrOW(~aUk0MRTjv=3mFH!O(t`yN3MviQk4E7_o&I#n~hlR1B z%!9=uU5zA2T%Yky%&j|IQvMkj>|I&MFxzeCG~o@WdAR;lDvYBr3sN-%bn?A?b?&o0 zJCzagtR=bDf&%wEfU;@p&A}pZpG=8%YnPJ3Ei%7I4lg$JyI4&w2#ABcJc@h=ABVf~ zlg@5Bf1B>9ES=3&C|m_!_2qI=)ITv1Ix{ys;N%fbSG4iRd!Uo<9xm{It|2DUdG`!0 zYGOwe9O=$(B5e&$5~3uVct9LEIt(FR@Z79wq<*{7f7ESh_nNOYztKo?TP1{j8qS)p z=IE~EBmC{H`E-QIHQ?uK0nj_#r4Fnm{E@_;`nK>=l14I<;yQP<77`YLc?KZ2dYz1) z+2B3+GgbaH(3afM#CKUnmp7BjEl4xGmoS0CoUa}ADztlr$UyoA$93L1Mh$ydQU5X0 z*P%2RK0?-v@4S$%Q)4t8hBbo}VtMiNy!QS}*7%Ol+us(loISUNfmdg%?N}~qUnd(h z(q74bbhlMiF|SXrv-Ia5&(+tSlp5|wM)Cw-ZSNZw`d?k$F8zMaym@fHEbu0Dw8W^9 z^jW=LqWBWO>{LWFl%NgMKTX}=U>L0rOB3<${q2T9T`es#we7H&E(m9b2yG`<$Fbl_ zS9r1ApWUfqe3)P+wa$sfrAKTwo`%EWYHA?tC&K@J62?*M*Ybp|-u=3gIrJuw@!D#; z<&f$pw5dRmaZqnfXG8; zgG_6LTZ;c5gHC&m=v$zaUqJ)$N+mfgP}*%z8Y-5;-l%qiYPEfDArnjzhv@{Mow2SN z{nyS%2+QqG*AD7@FGFOnq^!?^w>G<6f@OndknypqGR9Wxo_vy?*)x!UJ;zQy zyGfnFF>`~I8uK?YsJ@C?+y{#p)1tcr~`_*YDbZs^jLhV^xXE3zJ}(opUx zD#{BsMBi->p}?6)XOf{W=)z%pn%a>hVOI8^$MquN_!7bd3 zuNIzfF8p$VOWa7n{-Z~dJQ%`5{t)3T1Bac#)#^u!K~m#U?3c~CB9gAWXE6DI%oLHu zVX!!8A{$}#g_mXDI$Qo`F>X&(k^%VxHuV07oM#yWTr6QZ=j`FM>oQ_|{+#yIDHJ30 zh^ZA=I=}u)fHxfD4}3gc6{ewdniOhwi9w<|V__hV2>Gu6M7%$a%w^YBDKXX*UxDE! zEG%ELsNo0K1XtiPp5x0Oqm3@3w+0hhBH4x2W#$`+B}4(i%jWPHFz8|-W|da$NZ$aX zAn>y@VM|f)J+k^QHe}FV_}}PTYMd!2;)MHqiz0xC;)DdVU0+*En31d>|sMeHV9hR5t>Mnsq!w_ zGK#x~T54AgSf5u{axelY)EoMD4U4lq{B#v%*CXf1 z>E}P@dOPl#m1n{TI(SSt5A*YM*fmR*_^0;#cm>AAwdP-BoNxb`tq&W7IAZ)w$`(Lq zdcX9&FCW#i{5}W(4lod<>AnNR5;(H1z$H7p4#nvYkEL?%_B2>UFEbtC>E{UeydLF7 z2Izb?P*uGgVF{}5n4@fVll+?ba5P0G)TcuQOG`R(jJXvp4Ef?&*rl$(iSo&$NnsnI zBEvu6y(f|&(fWOAP#6GA#6g}*&KTKlOaAb+6u%PNUR@j$>M!5g1juJ=q>4DHD6Rt) z<0-|E$K&8OBJih<%y(D4Uz*_IQz_?c2C-&`!(va7C(ZZxvhXGj!T?}>-j^Xt*$L%p zJ)4I(Iuwi`Wf-F(lb;KMJ^aDg%QmP=0mUt_gIhI5K8DbAr`6wY{*A;Xp8J?;2!xn< zSgft&aBeS;v-h%-X|T#3B>^aoUaZYHTI zpCAV-h;HyFaCDak2L5|G-oc&NKd(F7qiDFLRXeC_vFN@iUbt?H4``bB^CBc?p03{hE?GI(I zKMd(C(n4{6d{=>L8gSD*TITIdb4GY4J7f(c^Oe9DEq@xJJ-L=wUP|~K_~FAjs?$6Y z<%tZWYLwI^`p0e-1TDKq!4;D4 zGnA(jIg7S*!Pw1<4+H*t7una$ea`W*ODJ%deVZH_J5kLL?=R*EqZ37E{t@5>VTR@- z1l-Ws5WoW~Z}4S3ZFk=nqf>{1THA^lgPNIwp~#QfB8{IQFwIiu=CVlP(@=Pi7CRpZ z2+42?OvUGFX6VGG`C?wgj|+*`Hln|=8Y5*)W}|c|;&uEj@&UPN!Onz7LmLPYKRnz* zuS~c1nX3bck$IDwqEP!I!-H7&k09USRe>VL)#EmN&^+8TWuTBIZ&$A!V7w;dT(9f$rzAB*NRV)Mol7zNb(iA_m`{@Z zS@cs@$9tD3@!B9mf(IbZnA^kPb9NwQAH(dEIjFbD%(T%=Lq6THBen{`i!e;(Fn*^0a^Q;QA2~ z>Dh_IQjyC%!`s33F|SorWS14U^&oH>9=9UYhUxb1+qo6%P(=Sjz^Si=X2{BZ9ibUd zYTBr9km5QS8P`cUMu4vVzpJz}-`Y;@v!&nCiVxMK1`go)FCSx)FEN0Jub4z?x@hsJ z(qA6KGse5`d=xx|qce{TnXvWPt9{7eqljypEMqB5$G7Ea-qCLN0D`?w*p8a@ z$A#a$Ps#*8if5LtzrFd z`;y^9PMElB)ybmNPDZx&5}JGI2&~L~+DOmb&~^t3HDrW;yQ8ZLectr;v~JOJ)c#qd zr+21&nzq{B9MxTO<~v5S0EI5SByd?Z=+t@nu_X}O>he|W?tNcp96`GxZ_(X7F;LEu zOEyu`uI`E9N7-YXVMO6S zrjDPu>YScxV%7cGZNq5*>pzLaXr?U88_Psalx7WkgC*-XiSd6jlIz2*W zQ!d-C4OV$+@)!hrgxKWgvfXAp2@O`L;!n{F|Ky_6Vuwq0Ov>)-`i%dw=WzrD*NidU zjFUcpcC7UwT&Nu{QUkMT3PJ|u5RKy(Zn|R0ui~Y#KxC{(iX(sD8s(8#-~X_I33{%W zxTqi!0e=9B!HL1e3B5D_vG2@(>^lSVzqbE@n{*F}uP}$p*dzamN{P#fRf^~b{vQ~B BA;SOw literal 0 HcmV?d00001 From 08d77852dbcd4913f08b3c5b104ec6b164587dd0 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 6 Dec 2017 12:23:46 +0100 Subject: [PATCH 02/36] Add data.world fields --- app/constants/constants.js | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index b12a3ec23..a97f95db0 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -179,7 +179,35 @@ export const CONNECTION_CONFIG = { 'value': 'secretAccessKey', 'type': 'password' } - ] // TODO - password options for apache drill + ], // TODO - password options for apache drill + [DIALECTS.DATA_WORLD]: [ + { + 'label': 'Owner', + 'value': 'owner', + 'type': 'text', + 'description': ` + User name and unique identifier of the creator of the dataset. + For example, in the URL: https://data.world/jonloyens/an-intro-to-dataworld-dataset, + jonloyens is the unique identifier of the dataset. + ` + }, + { + 'label': 'Id', + 'value': 'id', + 'type': 'text', + 'description': ` + Dataset unique identifier. + For example, in the URL: https://data.world/jonloyens/an-intro-to-dataworld-dataset, + an-intro-to-dataworld-dataset is the unique identifier of the dataset. + ` + }, + { + 'label': 'Token', + 'value': 'token', + 'type': 'text', + 'description': 'Your data.world token' + } + ] }; From 5858033d7fd3114e762087b6aadf29b6312ae438 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 6 Dec 2017 13:16:18 +0100 Subject: [PATCH 03/36] Add data.world sample credentials --- app/constants/constants.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/constants/constants.js b/app/constants/constants.js index a97f95db0..49876f246 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -408,5 +408,9 @@ export const SAMPLE_DBS = { sqlite: { dialect: 'sqlite', storage: `${__dirname}/plotly_datasets.db` + }, + [DIALECTS.DATA_WORLD]: { + owner: 'jonloyens', + id: 'an-intro-to-dataworld-dataset' } }; From 1d18c4ea2cd15e6db72cf1e8dda09b14e37dd7d7 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 7 Dec 2017 12:00:37 +0100 Subject: [PATCH 04/36] Connect data.world --- backend/persistent/datastores/DataWorld.js | 54 +++++++++++++++++++++ backend/persistent/datastores/Datastores.js | 3 ++ 2 files changed, 57 insertions(+) create mode 100644 backend/persistent/datastores/DataWorld.js diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js new file mode 100644 index 000000000..711be7442 --- /dev/null +++ b/backend/persistent/datastores/DataWorld.js @@ -0,0 +1,54 @@ +import AWS from 'aws-sdk'; +import {parseCSV} from '../../parse.js'; + +function createClient(connection) { + AWS.config.update({ + accessKeyId: connection.accessKeyId, + secretAccessKey: connection.secretAccessKey + }); + return new AWS.S3(); +} + + +export function connect(connection) { + return new Promise((resolve, reject) => { + resolve({success: true}); + }); +} + +/* + * Download a (csv) file from S3, parse it + * and return the results. + */ +export function query(key, connection) { + const {bucket} = connection; + const client = createClient(connection); + return new Promise((resolve, reject) => { + client.getObject({Bucket: bucket, Key: key}, (err, response) => { + if (err) { + reject(err); + return; + } + const textData = response.Body.toString(); + // TODO ^ handle binary files too + parseCSV(textData).then(resolve); + }); + }); +} + +/* + * List all of the files in an S3 bucket + */ +export function files(connection) { + const {bucket} = connection; + const client = createClient(connection); + return new Promise((resolve, reject) => { + client.listObjects({Bucket: bucket}, (err, data) => { + if (err) { + reject(err); + return; + } + resolve(data.Contents); + }); + }); +} diff --git a/backend/persistent/datastores/Datastores.js b/backend/persistent/datastores/Datastores.js index 075660593..c0657a17d 100644 --- a/backend/persistent/datastores/Datastores.js +++ b/backend/persistent/datastores/Datastores.js @@ -5,6 +5,7 @@ import * as ApacheDrill from './ApacheDrill'; import * as IbmDb2 from './ibmdb2'; import * as ApacheLivy from './livy'; import * as ApacheImpala from './impala'; +import * as DataWorld from './DataWorld'; import * as DatastoreMock from './datastoremock'; /* @@ -45,6 +46,8 @@ function getDatastoreClient(connection) { return ApacheImpala; } else if (dialect === 'ibm db2') { return IbmDb2; + } else if (dialect === 'data.world') { + return DataWorld; } return Sql; } From dc46f92c851a543aaa03985f2ab9c65732495553 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 7 Dec 2017 13:21:33 +0100 Subject: [PATCH 05/36] Connect to data.world with credentials --- backend/persistent/datastores/DataWorld.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 711be7442..5c15ec06d 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,5 +1,6 @@ import AWS from 'aws-sdk'; import {parseCSV} from '../../parse.js'; +import fetch from 'node-fetch'; function createClient(connection) { AWS.config.update({ @@ -12,9 +13,21 @@ function createClient(connection) { export function connect(connection) { return new Promise((resolve, reject) => { - resolve({success: true}); + fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.id}`, { + method: 'GET', + headers: { 'Authorization': `Bearer ${connection.token}` } + }) + .then(res => res.json()) + .then(json => { + // json.code is defined only when there is an error + if (json.code) { + reject(json); + } else { + resolve(); + } + }); }); -} + } /* * Download a (csv) file from S3, parse it From 313e110cde5256bd9fdef2306381e05d2a40c32d Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 7 Dec 2017 16:05:17 +0100 Subject: [PATCH 06/36] Add data.world tab text --- app/components/Settings/Tabs/Tab.react.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/components/Settings/Tabs/Tab.react.js b/app/components/Settings/Tabs/Tab.react.js index 70d3b6b94..96d066865 100644 --- a/app/components/Settings/Tabs/Tab.react.js +++ b/app/components/Settings/Tabs/Tab.react.js @@ -35,6 +35,8 @@ export default class ConnectionTab extends Component { label = `Elasticsearch (${connectionObject.host})`; } else if (connectionObject.dialect === DIALECTS.SQLITE) { label = connectionObject.storage; + } else if (connectionObject.dialect === DIALECTS.DATA_WORLD) { + label = `data.world (${connectionObject.owner}/${connectionObject.id})`; } else { label = `${connectionObject.database} (${connectionObject.username}@${connectionObject.host})`; } From f693cf5b653c5ce8f7228006bf0de3c7bf619b0a Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 13 Dec 2017 14:59:33 +0300 Subject: [PATCH 07/36] Add editor to data world connector --- app/constants/constants.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index 49876f246..beff6fc6f 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -27,7 +27,8 @@ export const SQL_DIALECTS_USING_EDITOR = [ 'sqlite', 'ibm db2', 'apache spark', - 'apache impala' + 'apache impala', + 'data.world' ]; const commonSqlOptions = [ @@ -136,7 +137,7 @@ export const CONNECTION_CONFIG = { [DIALECTS.ELASTICSEARCH]: [ {'label': 'Username', 'value': 'username', 'type': 'text', - 'description': ` + 'description': ` These credentials are used to authenticate Elasticsearch instances that are protected with HTTP Basic Auth. You can leave this blank if your instance does not have From 84b933b83f745e480197c9a5964a0d30bd23781f Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 13 Dec 2017 16:47:04 +0300 Subject: [PATCH 08/36] Display tables in data world dataset --- backend/persistent/datastores/DataWorld.js | 70 ++++++++-------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 5c15ec06d..be583ae35 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,16 +1,5 @@ -import AWS from 'aws-sdk'; -import {parseCSV} from '../../parse.js'; import fetch from 'node-fetch'; -function createClient(connection) { - AWS.config.update({ - accessKeyId: connection.accessKeyId, - secretAccessKey: connection.secretAccessKey - }); - return new AWS.S3(); -} - - export function connect(connection) { return new Promise((resolve, reject) => { fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.id}`, { @@ -23,45 +12,36 @@ export function connect(connection) { if (json.code) { reject(json); } else { - resolve(); + resolve(json); } }); }); } -/* - * Download a (csv) file from S3, parse it - * and return the results. - */ -export function query(key, connection) { - const {bucket} = connection; - const client = createClient(connection); - return new Promise((resolve, reject) => { - client.getObject({Bucket: bucket, Key: key}, (err, response) => { - if (err) { - reject(err); - return; - } - const textData = response.Body.toString(); - // TODO ^ handle binary files too - parseCSV(textData).then(resolve); - }); - }); +function getTables(allFiles) { + const csvFiles = allFiles.filter((file) => { + return /.csv$/.test(file.name); + }); + + return csvFiles; } -/* - * List all of the files in an S3 bucket - */ -export function files(connection) { - const {bucket} = connection; - const client = createClient(connection); - return new Promise((resolve, reject) => { - client.listObjects({Bucket: bucket}, (err, data) => { - if (err) { - reject(err); - return; - } - resolve(data.Contents); - }); - }); +export function schemas(connection) { + return connect(connection) + .then((json) => { + const tables = getTables(json.files); + const tableNames = tables.map((table) => { + const tableName = table.name; + return tableName.substring(0, tableName.length - 4); + }); + const rows = tableNames.map((tableName) => { + return [tableName]; + }); + return { + columnnames: ['?column?', 'column_name', 'data_type'], + rows + }; + }).catch(err => { + throw new Error(err); + }); } From 831d33a1594189937c3280c385a66ca30fd50022 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 13 Dec 2017 18:51:49 +0300 Subject: [PATCH 09/36] [WIP] Get schema of each table in dataset --- backend/persistent/datastores/DataWorld.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index be583ae35..80bb1a9e8 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -18,6 +18,20 @@ export function connect(connection) { }); } +function getSchema(fileName, connection) { + fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.id}`, { + method: 'POST', + headers: { 'Accept': 'application/json', 'Authorization': `Bearer ${connection.token}` } + body: JSON.stringify({ + query: `SELECT * FROM ${fileName} LIMIT 1` + }) + }) + .then(res => res.json()) + .then(json => { + console.log(json) + }); +} + function getTables(allFiles) { const csvFiles = allFiles.filter((file) => { return /.csv$/.test(file.name); @@ -34,6 +48,9 @@ export function schemas(connection) { const tableName = table.name; return tableName.substring(0, tableName.length - 4); }); + tableNames.map((name) => { + getSchema(name); + }); const rows = tableNames.map((tableName) => { return [tableName]; }); From 842132e980165b7e83354425b89c9fcb78e1c791 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 12:22:18 +0300 Subject: [PATCH 10/36] Rename id to identifier to prevent a clash --- app/constants/constants.js | 6 +++--- backend/persistent/datastores/DataWorld.js | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index beff6fc6f..70ee4fd57 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -193,8 +193,8 @@ export const CONNECTION_CONFIG = { ` }, { - 'label': 'Id', - 'value': 'id', + 'label': 'Identifier', + 'value': 'identifier', 'type': 'text', 'description': ` Dataset unique identifier. @@ -412,6 +412,6 @@ export const SAMPLE_DBS = { }, [DIALECTS.DATA_WORLD]: { owner: 'jonloyens', - id: 'an-intro-to-dataworld-dataset' + identifier: 'an-intro-to-dataworld-dataset' } }; diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 80bb1a9e8..146ace27c 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,8 +1,9 @@ import fetch from 'node-fetch'; export function connect(connection) { + console.log('This is the connection', connection); return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.id}`, { + fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { method: 'GET', headers: { 'Authorization': `Bearer ${connection.token}` } }) @@ -12,7 +13,7 @@ export function connect(connection) { if (json.code) { reject(json); } else { - resolve(json); + resolve(); } }); }); @@ -21,7 +22,7 @@ export function connect(connection) { function getSchema(fileName, connection) { fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.id}`, { method: 'POST', - headers: { 'Accept': 'application/json', 'Authorization': `Bearer ${connection.token}` } + headers: { 'Accept': 'application/json', 'Authorization': `Bearer ${connection.token}` }, body: JSON.stringify({ query: `SELECT * FROM ${fileName} LIMIT 1` }) From b27d62cba7daaff8e004fed0a5abb8be1cdeabad Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 12:46:02 +0300 Subject: [PATCH 11/36] Finish adding UI changes --- app/actions/sessions.js | 2 +- app/components/Settings/Settings.react.js | 3 ++- app/components/Settings/Tabs/Tab.react.js | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/actions/sessions.js b/app/actions/sessions.js index 64204126b..ceac1271d 100644 --- a/app/actions/sessions.js +++ b/app/actions/sessions.js @@ -354,4 +354,4 @@ export function setConnectionNeedToBeSaved(tabId, bool) { } }); }; -} +} \ No newline at end of file diff --git a/app/components/Settings/Settings.react.js b/app/components/Settings/Settings.react.js index 2b15aff0f..e22abdfbb 100644 --- a/app/components/Settings/Settings.react.js +++ b/app/components/Settings/Settings.react.js @@ -199,7 +199,8 @@ class Settings extends Component { DIALECTS.APACHE_SPARK, DIALECTS.IBM_DB2, DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES, - DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE + DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE, + DIALECTS.DATA_WORLD ])) { if (connectRequest.status === 200 && !tablesRequest.status) { this.setState({editMode: false}); diff --git a/app/components/Settings/Tabs/Tab.react.js b/app/components/Settings/Tabs/Tab.react.js index 96d066865..800bd9ac1 100644 --- a/app/components/Settings/Tabs/Tab.react.js +++ b/app/components/Settings/Tabs/Tab.react.js @@ -36,7 +36,7 @@ export default class ConnectionTab extends Component { } else if (connectionObject.dialect === DIALECTS.SQLITE) { label = connectionObject.storage; } else if (connectionObject.dialect === DIALECTS.DATA_WORLD) { - label = `data.world (${connectionObject.owner}/${connectionObject.id})`; + label = `data.world (${connectionObject.owner}/${connectionObject.identifier})`; } else { label = `${connectionObject.database} (${connectionObject.username}@${connectionObject.host})`; } From 5473fdee15990c06b356f606724603a14e6bc2ee Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 13:22:20 +0300 Subject: [PATCH 12/36] Implement tables function --- backend/persistent/datastores/DataWorld.js | 85 +++++++++------------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 146ace27c..aa7d67627 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,65 +1,48 @@ import fetch from 'node-fetch'; export function connect(connection) { - console.log('This is the connection', connection); - return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { - method: 'GET', - headers: { 'Authorization': `Bearer ${connection.token}` } - }) - .then(res => res.json()) - .then(json => { - // json.code is defined only when there is an error - if (json.code) { - reject(json); - } else { - resolve(); - } - }); - }); - } - -function getSchema(fileName, connection) { - fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.id}`, { - method: 'POST', - headers: { 'Accept': 'application/json', 'Authorization': `Bearer ${connection.token}` }, - body: JSON.stringify({ - query: `SELECT * FROM ${fileName} LIMIT 1` + return new Promise((resolve, reject) => { + fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { + method: 'GET', + headers: { 'Authorization': `Bearer ${connection.token}` } }) - }) - .then(res => res.json()) - .then(json => { - console.log(json) + .then(res => res.json()) + .then(json => { + // json.code is defined only when there is an error + if (json.code) { + reject(json); + } else { + resolve(); + } + }); }); -} + } function getTables(allFiles) { const csvFiles = allFiles.filter((file) => { return /.csv$/.test(file.name); }); - - return csvFiles; + const tablesNames = csvFiles.map((table) => { + return table.name.substring(0, table.name.length - 4).toLowerCase(); + }); + + return tablesNames; } -export function schemas(connection) { - return connect(connection) - .then((json) => { - const tables = getTables(json.files); - const tableNames = tables.map((table) => { - const tableName = table.name; - return tableName.substring(0, tableName.length - 4); - }); - tableNames.map((name) => { - getSchema(name); - }); - const rows = tableNames.map((tableName) => { - return [tableName]; - }); - return { - columnnames: ['?column?', 'column_name', 'data_type'], - rows - }; - }).catch(err => { - throw new Error(err); +export function tables(connection) { + return new Promise((resolve, reject) => { + fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { + method: 'GET', + headers: { 'Authorization': `Bearer ${connection.token}` } + }) + .then(res => res.json()) + .then(json => { + // json.code is defined only when there is an error + if (json.code) { + reject(json); + } else { + resolve(getTables(json.files)); + } + }); }); } From e9535d6c110489051834bd429f9c070e12c9afec Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 14:26:20 +0300 Subject: [PATCH 13/36] Retrieve table schemas --- backend/persistent/datastores/DataWorld.js | 49 ++++++++++++++++++++++ backend/persistent/datastores/impala.js | 1 + 2 files changed, 50 insertions(+) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index aa7d67627..a5143f57a 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -46,3 +46,52 @@ export function tables(connection) { }); }); } + +function getSchema(connection, table) { + console.log('The table', table); + const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); + fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: params + }) + .then(res => { + return res.json(); + }) + .then(json => { + return json.fields; + }); +} + +export function schemas(connection) { + return new Promise((resolve, reject) => { + fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { + method: 'GET', + headers: { 'Authorization': `Bearer ${connection.token}` } + }) + .then(res => res.json()) + .then(json => { + // json.code is defined only when there is an error + if (json.code) { + reject(json); + } else { + const tableSchemas = getTables(json.files).map(table => { + const rs = getSchema(connection, table); + return rs; + }); + + console.log(tableSchemas); + } + }); + resolve({ + columnNames: [ 'tablename', 'column_name', 'data_type' ], + rows: [ + [ 'plotly.alcohol_consumption_by_country_2010', 'loc', 'string' ], + [ 'plotly.alcohol_consumption_by_country_2010', 'alcohol', 'double' ] + ] + }); + }); +} diff --git a/backend/persistent/datastores/impala.js b/backend/persistent/datastores/impala.js index 0d905774f..3b473045e 100644 --- a/backend/persistent/datastores/impala.js +++ b/backend/persistent/datastores/impala.js @@ -67,6 +67,7 @@ export function schemas(connection) { // The results are nested inside a list, so we need to un-nest first: const rows = unnest(res); + console.log('Imapala schema', {columnnames, rows}); return {columnnames, rows}; }).catch(err => { Logger.log(err); From 742ae6928273962537f6658a2a7a0065fbf24ecb Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 17:56:51 +0300 Subject: [PATCH 14/36] Implement schema function --- backend/persistent/datastores/DataWorld.js | 48 +++++++++++----------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index a5143f57a..e32f19672 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -48,22 +48,23 @@ export function tables(connection) { } function getSchema(connection, table) { - console.log('The table', table); - const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); - fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' - }, - body: params - }) + return new Promise((resolve, reject) => { + const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); + fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: params + }) .then(res => { return res.json(); }) .then(json => { - return json.fields; + resolve({table, fields: json[0].fields }); }); + }); } export function schemas(connection) { @@ -78,20 +79,21 @@ export function schemas(connection) { if (json.code) { reject(json); } else { - const tableSchemas = getTables(json.files).map(table => { - const rs = getSchema(connection, table); - return rs; + const promises = getTables(json.files).map(table => { + return getSchema(connection, table).then((schema) => { + return schema.fields.map((field) => { + return [schema.table, field.name, field.type]; + }); + }); + }); + Promise.all(promises).then((tableSchemas) => { + resolve({ + columnNames: [ 'tablename', 'column_name', 'data_type' ], + rows: [].concat.apply([], tableSchemas) + }); }); - - console.log(tableSchemas); } }); - resolve({ - columnNames: [ 'tablename', 'column_name', 'data_type' ], - rows: [ - [ 'plotly.alcohol_consumption_by_country_2010', 'loc', 'string' ], - [ 'plotly.alcohol_consumption_by_country_2010', 'alcohol', 'double' ] - ] - }); + }); } From 5ff119d0161b18af9f951aa9e8227ba6659373d2 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Thu, 14 Dec 2017 18:48:28 +0300 Subject: [PATCH 15/36] Implement query function --- backend/persistent/datastores/DataWorld.js | 30 ++++++++++++++++++++++ backend/persistent/datastores/impala.js | 1 - 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index e32f19672..e87ca7878 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -97,3 +97,33 @@ export function schemas(connection) { }); } + +export function query(queryStmt, connection) { + return new Promise((resolve, reject) => { + const params = encodeURIComponent('query') + '=' + encodeURIComponent(queryStmt); + fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: params + }) + .then(res => { + return res.json(); + }) + .then(json => { + const fields = json[0].fields; + const columnnames = fields.map((field) => { + return field.name; + }); + const rows = json.slice(1).map((row) => { + return Object.values(row); + }); + resolve({ + columnnames, + rows + }); + }); + }); +} diff --git a/backend/persistent/datastores/impala.js b/backend/persistent/datastores/impala.js index 3b473045e..0d905774f 100644 --- a/backend/persistent/datastores/impala.js +++ b/backend/persistent/datastores/impala.js @@ -67,7 +67,6 @@ export function schemas(connection) { // The results are nested inside a list, so we need to un-nest first: const rows = unnest(res); - console.log('Imapala schema', {columnnames, rows}); return {columnnames, rows}; }).catch(err => { Logger.log(err); From 11fae2ee236b3dd54f2ebe8123efa082ad7971eb Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Fri, 15 Dec 2017 16:34:36 +0300 Subject: [PATCH 16/36] Replace - with _ on queries --- backend/persistent/datastores/DataWorld.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index e87ca7878..8c9677fb3 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -18,7 +18,7 @@ export function connect(connection) { }); } -function getTables(allFiles) { +function getCsvs(allFiles) { const csvFiles = allFiles.filter((file) => { return /.csv$/.test(file.name); }); @@ -41,14 +41,15 @@ export function tables(connection) { if (json.code) { reject(json); } else { - resolve(getTables(json.files)); + resolve(getCsvs(json.files)); } }); }); } -function getSchema(connection, table) { +function getSchema(connection, csvFile) { return new Promise((resolve, reject) => { + const table = csvFile.replace(/-/g, '_') const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { method: 'POST', @@ -79,8 +80,8 @@ export function schemas(connection) { if (json.code) { reject(json); } else { - const promises = getTables(json.files).map(table => { - return getSchema(connection, table).then((schema) => { + const promises = getCsvs(json.files).map(csvFile => { + return getSchema(connection, csvFile).then((schema) => { return schema.fields.map((field) => { return [schema.table, field.name, field.type]; }); @@ -100,7 +101,8 @@ export function schemas(connection) { export function query(queryStmt, connection) { return new Promise((resolve, reject) => { - const params = encodeURIComponent('query') + '=' + encodeURIComponent(queryStmt); + const query = queryStmt.replace(/-/g, '_'); + const params = encodeURIComponent('query') + '=' + encodeURIComponent(query); fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { method: 'POST', headers: { From e0424ee00eded4d7350326e785783bbf36141064 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 2 Jan 2018 15:13:19 +0300 Subject: [PATCH 17/36] Use query to get all tables --- backend/persistent/datastores/DataWorld.js | 34 ++++++++-------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 8c9677fb3..4e643e331 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -70,32 +70,22 @@ function getSchema(connection, csvFile) { export function schemas(connection) { return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { - method: 'GET', - headers: { 'Authorization': `Bearer ${connection.token}` } - }) - .then(res => res.json()) - .then(json => { - // json.code is defined only when there is an error - if (json.code) { - reject(json); - } else { - const promises = getCsvs(json.files).map(csvFile => { - return getSchema(connection, csvFile).then((schema) => { - return schema.fields.map((field) => { - return [schema.table, field.name, field.type]; - }); + query('SELECT * FROM Tables', connection).then((res) => { + const promises = res.rows.map((table) => { + return getSchema(connection, table[0]).then((schema) => { + return schema.fields.map((field) => { + return [schema.table, field.name, field.type]; }); }); - Promise.all(promises).then((tableSchemas) => { - resolve({ - columnNames: [ 'tablename', 'column_name', 'data_type' ], - rows: [].concat.apply([], tableSchemas) - }); + }); + + Promise.all(promises).then((tableSchemas) => { + resolve({ + columnNames: [ 'tablename', 'column_name', 'data_type' ], + rows: [].concat.apply([], tableSchemas) }); - } + }); }); - }); } From a18eee42ad5adb2b327767776c5a5ee4226313b1 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 2 Jan 2018 15:31:54 +0300 Subject: [PATCH 18/36] Remove redundant code --- backend/persistent/datastores/DataWorld.js | 28 +++++++++------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 4e643e331..11d0dc851 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -31,25 +31,19 @@ function getCsvs(allFiles) { export function tables(connection) { return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { - method: 'GET', - headers: { 'Authorization': `Bearer ${connection.token}` } - }) - .then(res => res.json()) - .then(json => { - // json.code is defined only when there is an error - if (json.code) { - reject(json); - } else { - resolve(getCsvs(json.files)); - } + query('SELECT * FROM Tables', connection).then((res) => { + const allTables = res.rows.map((table) => { + return table[0]; + }); + + resolve(allTables); }); }); } -function getSchema(connection, csvFile) { +function getSchema(connection, tableName) { return new Promise((resolve, reject) => { - const table = csvFile.replace(/-/g, '_') + const table = tableName.replace(/-/g, '_') const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { method: 'POST', @@ -70,9 +64,9 @@ function getSchema(connection, csvFile) { export function schemas(connection) { return new Promise((resolve, reject) => { - query('SELECT * FROM Tables', connection).then((res) => { - const promises = res.rows.map((table) => { - return getSchema(connection, table[0]).then((schema) => { + tables(connection).then((allTables) => { + const promises = allTables.map((table) => { + return getSchema(connection, table).then((schema) => { return schema.fields.map((field) => { return [schema.table, field.name, field.type]; }); From 78ab1e01ca4911c1b4d31d3104d1d64186916ecc Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 2 Jan 2018 16:14:44 +0300 Subject: [PATCH 19/36] Use dataset URL --- app/constants/constants.js | 23 ++++-------------- backend/persistent/datastores/DataWorld.js | 28 +++++++++++----------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index 70ee4fd57..8764a258a 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -183,24 +183,10 @@ export const CONNECTION_CONFIG = { ], // TODO - password options for apache drill [DIALECTS.DATA_WORLD]: [ { - 'label': 'Owner', - 'value': 'owner', + 'label': 'Dataset URL', + 'value': 'url', 'type': 'text', - 'description': ` - User name and unique identifier of the creator of the dataset. - For example, in the URL: https://data.world/jonloyens/an-intro-to-dataworld-dataset, - jonloyens is the unique identifier of the dataset. - ` - }, - { - 'label': 'Identifier', - 'value': 'identifier', - 'type': 'text', - 'description': ` - Dataset unique identifier. - For example, in the URL: https://data.world/jonloyens/an-intro-to-dataworld-dataset, - an-intro-to-dataworld-dataset is the unique identifier of the dataset. - ` + 'description': 'The dataset\'s URL on data.world' }, { 'label': 'Token', @@ -411,7 +397,6 @@ export const SAMPLE_DBS = { storage: `${__dirname}/plotly_datasets.db` }, [DIALECTS.DATA_WORLD]: { - owner: 'jonloyens', - identifier: 'an-intro-to-dataworld-dataset' + url: 'https://data.world/jonloyens/an-intro-to-dataworld-dataset' } }; diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 11d0dc851..daa87f469 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,8 +1,18 @@ import fetch from 'node-fetch'; +import url from 'url'; + +function parseUrl(datasetUrl) { + const pathnameArray = url.parse(datasetUrl).pathname.split('/'); + return { + owner: pathnameArray[1], + id: pathnameArray[2] + }; +} export function connect(connection) { + const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${connection.owner}/${connection.identifier}`, { + fetch(`https://api.data.world/v0/datasets/${owner}/${id}`, { method: 'GET', headers: { 'Authorization': `Bearer ${connection.token}` } }) @@ -18,17 +28,6 @@ export function connect(connection) { }); } -function getCsvs(allFiles) { - const csvFiles = allFiles.filter((file) => { - return /.csv$/.test(file.name); - }); - const tablesNames = csvFiles.map((table) => { - return table.name.substring(0, table.name.length - 4).toLowerCase(); - }); - - return tablesNames; -} - export function tables(connection) { return new Promise((resolve, reject) => { query('SELECT * FROM Tables', connection).then((res) => { @@ -42,10 +41,11 @@ export function tables(connection) { } function getSchema(connection, tableName) { + const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - const table = tableName.replace(/-/g, '_') + const table = tableName.replace(/-/g, '_'); const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); - fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { + fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', headers: { 'Authorization': `Bearer ${connection.token}`, From 1c7b19d58627b000863148c2f823ea74547700fd Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 2 Jan 2018 16:52:57 +0300 Subject: [PATCH 20/36] Get tab label from provided URL --- app/components/Settings/Tabs/Tab.react.js | 9 ++++++++- app/constants/constants.js | 2 +- backend/persistent/datastores/DataWorld.js | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/components/Settings/Tabs/Tab.react.js b/app/components/Settings/Tabs/Tab.react.js index 800bd9ac1..c7ff8e083 100644 --- a/app/components/Settings/Tabs/Tab.react.js +++ b/app/components/Settings/Tabs/Tab.react.js @@ -36,7 +36,14 @@ export default class ConnectionTab extends Component { } else if (connectionObject.dialect === DIALECTS.SQLITE) { label = connectionObject.storage; } else if (connectionObject.dialect === DIALECTS.DATA_WORLD) { - label = `data.world (${connectionObject.owner}/${connectionObject.identifier})`; + const parser = document.createElement('a'); + parser.href = connectionObject.url; + const pathNames = parser.pathname.split('/'); + if (pathNames.length >= 3) { + label = `data.world (${pathNames[1]}/${pathNames[2]})`; + } else { + label = 'data.world (/)'; + } } else { label = `${connectionObject.database} (${connectionObject.username}@${connectionObject.host})`; } diff --git a/app/constants/constants.js b/app/constants/constants.js index 8764a258a..ce51adf1d 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -137,7 +137,7 @@ export const CONNECTION_CONFIG = { [DIALECTS.ELASTICSEARCH]: [ {'label': 'Username', 'value': 'username', 'type': 'text', - 'description': ` + 'description': ` These credentials are used to authenticate Elasticsearch instances that are protected with HTTP Basic Auth. You can leave this blank if your instance does not have diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index daa87f469..17cc4d1f7 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -1,6 +1,8 @@ import fetch from 'node-fetch'; import url from 'url'; +import Logger from '../../logger'; + function parseUrl(datasetUrl) { const pathnameArray = url.parse(datasetUrl).pathname.split('/'); return { @@ -25,6 +27,10 @@ export function connect(connection) { resolve(); } }); + }) + .catch(err => { + Logger.log(err); + throw new Error(err); }); } From 5b58e4115e820d9ceaac4ed3f1082ae919a6e606 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 2 Jan 2018 17:12:31 +0300 Subject: [PATCH 21/36] Change sample dataset URL --- app/constants/constants.js | 2 +- backend/persistent/datastores/DataWorld.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index ce51adf1d..6164880f6 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -397,6 +397,6 @@ export const SAMPLE_DBS = { storage: `${__dirname}/plotly_datasets.db` }, [DIALECTS.DATA_WORLD]: { - url: 'https://data.world/jonloyens/an-intro-to-dataworld-dataset' + url: 'https://data.world/rflprr/reported-lyme-disease-cases-by-state' } }; diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 17cc4d1f7..0e6972c5d 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -89,11 +89,12 @@ export function schemas(connection) { }); } -export function query(queryStmt, connection) { +export function query(query, connection) { + const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - const query = queryStmt.replace(/-/g, '_'); - const params = encodeURIComponent('query') + '=' + encodeURIComponent(query); - fetch(`https://api.data.world/v0/sql/${connection.owner}/${connection.identifier}?includeTableSchema=true`, { + const queryStatement = `${query.replace(/-/g, '_')}`; + const params = encodeURIComponent('query') + '=' + encodeURIComponent(queryStatement); + fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', headers: { 'Authorization': `Bearer ${connection.token}`, From 3148d225f01d6a6fd3ae87b80775786973aefee2 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 12:54:36 +0300 Subject: [PATCH 22/36] Update text on connection tab --- app/actions/sessions.js | 2 +- app/constants/constants.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/actions/sessions.js b/app/actions/sessions.js index ceac1271d..64204126b 100644 --- a/app/actions/sessions.js +++ b/app/actions/sessions.js @@ -354,4 +354,4 @@ export function setConnectionNeedToBeSaved(tabId, bool) { } }); }; -} \ No newline at end of file +} diff --git a/app/constants/constants.js b/app/constants/constants.js index 6164880f6..2fde4c5f1 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -183,16 +183,16 @@ export const CONNECTION_CONFIG = { ], // TODO - password options for apache drill [DIALECTS.DATA_WORLD]: [ { - 'label': 'Dataset URL', + 'label': 'Dataset/Project URL', 'value': 'url', 'type': 'text', - 'description': 'The dataset\'s URL on data.world' + 'description': 'The URL of the dataset or project on data.world' }, { 'label': 'Token', 'value': 'token', 'type': 'text', - 'description': 'Your data.world token' + 'description': 'Your data.world token. Can be obtained from https://data.world/settings/advanced' } ] }; From 69a73b54d66716831e22316e208181fd370ed702 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 14:43:46 +0300 Subject: [PATCH 23/36] Get database name from supplied URL --- .../Settings/Preview/TableTree.react.js | 16 +++++++++++++--- app/components/Settings/Tabs/Tab.react.js | 5 ++--- app/utils/utils.js | 8 ++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/app/components/Settings/Preview/TableTree.react.js b/app/components/Settings/Preview/TableTree.react.js index b267ab05d..2d268fa5b 100644 --- a/app/components/Settings/Preview/TableTree.react.js +++ b/app/components/Settings/Preview/TableTree.react.js @@ -4,6 +4,7 @@ import TreeView from 'react-treeview'; import {isEmpty, has} from 'ramda'; import {DIALECTS} from '../../../constants/constants'; +import {getPathNames} from '../../../utils/utils'; const BASENAME_RE = /[^\\/]+$/; @@ -26,6 +27,17 @@ class TableTree extends Component { }) } + getLabel(connectionObject) { + switch (connectionObject.dialect) { + case DIALECTS.SQLITE: + return BASENAME_RE.exec(connectionObject.storage)[0] || connectionObject.storage; + case DIALECTS.DATA_WORLD: + return getPathNames(connectionObject.url)[2]; + default: + return connectionObject.database; + } + } + storeSchemaTree() { const {schemaRequest, getSqlSchema, updatePreview} = this.props; @@ -84,9 +96,7 @@ class TableTree extends Component { return (
{'Updating'}
); } - const label = (this.props.connectionObject.dialect === DIALECTS.SQLITE) ? - BASENAME_RE.exec(this.props.connectionObject.storage)[0] || this.props.connectionObject.storage : - this.props.connectionObject.database; + const label = this.getLabel(this.props.connectionObject); const labelNode = {label}; return ( diff --git a/app/components/Settings/Tabs/Tab.react.js b/app/components/Settings/Tabs/Tab.react.js index c7ff8e083..21a65d5ff 100644 --- a/app/components/Settings/Tabs/Tab.react.js +++ b/app/components/Settings/Tabs/Tab.react.js @@ -2,6 +2,7 @@ import React, {Component} from 'react'; import PropTypes from 'prop-types'; import classnames from 'classnames'; import {LOGOS, DIALECTS} from '../../../constants/constants'; +import {getPathNames} from '../../../utils/utils'; export default class ConnectionTab extends Component { constructor(props) { @@ -36,9 +37,7 @@ export default class ConnectionTab extends Component { } else if (connectionObject.dialect === DIALECTS.SQLITE) { label = connectionObject.storage; } else if (connectionObject.dialect === DIALECTS.DATA_WORLD) { - const parser = document.createElement('a'); - parser.href = connectionObject.url; - const pathNames = parser.pathname.split('/'); + const pathNames = getPathNames(connectionObject.url); if (pathNames.length >= 3) { label = `data.world (${pathNames[1]}/${pathNames[2]})`; } else { diff --git a/app/utils/utils.js b/app/utils/utils.js index e3d4f6488..5b030271e 100644 --- a/app/utils/utils.js +++ b/app/utils/utils.js @@ -49,3 +49,11 @@ export function homeUrl() { '/external-data-connector' : ''; } + +export function getPathNames(url) { + const parser = document.createElement('a'); + parser.href = url; + const pathNames = parser.pathname.split('/'); + + return pathNames; +} From 4f44902599ab33245eb0d4486e89c7ad14c64050 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 16:27:27 +0300 Subject: [PATCH 24/36] Add preview query --- app/constants/constants.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/constants/constants.js b/app/constants/constants.js index 2fde4c5f1..740dd5bb2 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -224,6 +224,7 @@ export function PREVIEW_QUERY (dialect, table, database = '') { case DIALECTS.SQLITE: case DIALECTS.MARIADB: case DIALECTS.POSTGRES: + case DIALECTS.DATA_WORLD: case DIALECTS.REDSHIFT: return `SELECT * FROM ${table} LIMIT 1000`; case DIALECTS.MSSQL: From 3d1b53c066789223b0d4ba26a31e56df37f82dde Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 17:12:34 +0300 Subject: [PATCH 25/36] Error handling --- backend/persistent/datastores/DataWorld.js | 34 +++++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 0e6972c5d..586c7517a 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -26,11 +26,11 @@ export function connect(connection) { } else { resolve(); } + }) + .catch(err => { + Logger.log(err); + throw new Error(err); }); - }) - .catch(err => { - Logger.log(err); - throw new Error(err); }); } @@ -42,6 +42,9 @@ export function tables(connection) { }); resolve(allTables); + }) + .catch(err => { + reject(err); }); }); } @@ -50,7 +53,7 @@ function getSchema(connection, tableName) { const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { const table = tableName.replace(/-/g, '_'); - const params = encodeURIComponent('query') + '=' + encodeURIComponent(`select * from ${table} limit 1`); + const params = encodeURIComponent('query') + '=' + encodeURIComponent(`SELECT * FROM ${table} LIMIT 1`); fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', headers: { @@ -64,12 +67,15 @@ function getSchema(connection, tableName) { }) .then(json => { resolve({table, fields: json[0].fields }); + }) + .catch(err => { + reject(err); }); }); } export function schemas(connection) { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { tables(connection).then((allTables) => { const promises = allTables.map((table) => { return getSchema(connection, table).then((schema) => { @@ -84,15 +90,23 @@ export function schemas(connection) { columnNames: [ 'tablename', 'column_name', 'data_type' ], rows: [].concat.apply([], tableSchemas) }); + }) + .catch(err => { + Logger.log(err); + throw new Error(err); }); + }) + .catch(err => { + Logger.log(err); + throw new Error(err); }); }); } -export function query(query, connection) { +export function query(queryString, connection) { const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - const queryStatement = `${query.replace(/-/g, '_')}`; + const queryStatement = `${queryString.replace(/-/g, '_')}`; const params = encodeURIComponent('query') + '=' + encodeURIComponent(queryStatement); fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', @@ -117,6 +131,10 @@ export function query(query, connection) { columnnames, rows }); + }) + .catch(err => { + Logger.log(err); + throw new Error(err); }); }); } From 9b510f9e44cd609b1903840265f9f8f8318007b5 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 18:47:41 +0300 Subject: [PATCH 26/36] Use single API call in schema function --- backend/persistent/datastores/DataWorld.js | 48 +++++++--------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 586c7517a..4a98131de 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -49,11 +49,10 @@ export function tables(connection) { }); } -function getSchema(connection, tableName) { +export function schemas(connection) { const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - const table = tableName.replace(/-/g, '_'); - const params = encodeURIComponent('query') + '=' + encodeURIComponent(`SELECT * FROM ${table} LIMIT 1`); + const params = `${encodeURIComponent('query')}=${encodeURIComponent('SELECT * FROM TableColumns')}`; fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', headers: { @@ -61,39 +60,22 @@ function getSchema(connection, tableName) { 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' }, body: params - }) - .then(res => { + }).then(res => { return res.json(); }) .then(json => { - resolve({table, fields: json[0].fields }); - }) - .catch(err => { - reject(err); - }); - }); -} - -export function schemas(connection) { - return new Promise((resolve) => { - tables(connection).then((allTables) => { - const promises = allTables.map((table) => { - return getSchema(connection, table).then((schema) => { - return schema.fields.map((field) => { - return [schema.table, field.name, field.type]; - }); - }); + const rows = json.splice(1).map(table => { + // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer + const dataType = /#(.*)/.exec(table.columnDatatype)[1]; + return [ + table.tableId, + table.columnName, + dataType + ]; }); - - Promise.all(promises).then((tableSchemas) => { - resolve({ - columnNames: [ 'tablename', 'column_name', 'data_type' ], - rows: [].concat.apply([], tableSchemas) - }); - }) - .catch(err => { - Logger.log(err); - throw new Error(err); + resolve({ + columnNames: [ 'tablename', 'column_name', 'data_type' ], + rows }); }) .catch(err => { @@ -107,7 +89,7 @@ export function query(queryString, connection) { const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { const queryStatement = `${queryString.replace(/-/g, '_')}`; - const params = encodeURIComponent('query') + '=' + encodeURIComponent(queryStatement); + const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryStatement)}`; fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', headers: { From 17fb8e69ccd1c7c3a281bd43bc4d329c0850a5ef Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 20:46:41 +0300 Subject: [PATCH 27/36] Add a user agent to all API requests --- backend/persistent/datastores/DataWorld.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 4a98131de..e3cf531ef 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -16,7 +16,11 @@ export function connect(connection) { return new Promise((resolve, reject) => { fetch(`https://api.data.world/v0/datasets/${owner}/${id}`, { method: 'GET', - headers: { 'Authorization': `Bearer ${connection.token}` } + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` + } }) .then(res => res.json()) .then(json => { @@ -57,7 +61,8 @@ export function schemas(connection) { method: 'POST', headers: { 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` }, body: params }).then(res => { @@ -94,7 +99,8 @@ export function query(queryString, connection) { method: 'POST', headers: { 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` }, body: params }) From 563112f8a485f391c5bed818802c69c318ba500d Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 3 Jan 2018 21:23:09 +0300 Subject: [PATCH 28/36] Remove duplicate code --- backend/persistent/datastores/DataWorld.js | 34 ++++++++-------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index e3cf531ef..809146796 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -54,30 +54,21 @@ export function tables(connection) { } export function schemas(connection) { - const { owner, id } = parseUrl(connection.url); - return new Promise((resolve, reject) => { - const params = `${encodeURIComponent('query')}=${encodeURIComponent('SELECT * FROM TableColumns')}`; - fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', - 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` - }, - body: params - }).then(res => { - return res.json(); - }) - .then(json => { - const rows = json.splice(1).map(table => { + return new Promise((resolve) => { + query('SELECT * FROM TableColumns', connection).then((res) => { + const rows = res.rows.splice(1).map((table) => { + const tableName = table[0]; + const columnName = table[3]; // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer - const dataType = /#(.*)/.exec(table.columnDatatype)[1]; + const columnDataType = /#(.*)/.exec(table[6])[1]; + return [ - table.tableId, - table.columnName, - dataType + tableName, + columnName, + columnDataType ]; }); + resolve({ columnNames: [ 'tablename', 'column_name', 'data_type' ], rows @@ -121,8 +112,7 @@ export function query(queryString, connection) { }); }) .catch(err => { - Logger.log(err); - throw new Error(err); + reject(err); }); }); } From a0d702114cc1ef42b70fc673e552a39f3d041e26 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Mon, 8 Jan 2018 15:17:52 +0300 Subject: [PATCH 29/36] Add data.world tests --- app/constants/constants.js | 2 +- backend/persistent/datastores/DataWorld.js | 4 +- package.json | 1 + test/backend/datastores.dataworld.spec.js | 116 +++++++++++++ test/backend/utils.js | 189 +++++++++++++++++++++ yarn.lock | 26 ++- 6 files changed, 331 insertions(+), 7 deletions(-) create mode 100644 test/backend/datastores.dataworld.spec.js diff --git a/app/constants/constants.js b/app/constants/constants.js index 740dd5bb2..2e3d79abf 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -192,7 +192,7 @@ export const CONNECTION_CONFIG = { 'label': 'Token', 'value': 'token', 'type': 'text', - 'description': 'Your data.world token. Can be obtained from https://data.world/settings/advanced' + 'description': 'Your data.world token. It can be obtained from https://data.world/settings/advanced' } ] }; diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js index 809146796..19746de79 100644 --- a/backend/persistent/datastores/DataWorld.js +++ b/backend/persistent/datastores/DataWorld.js @@ -14,7 +14,7 @@ function parseUrl(datasetUrl) { export function connect(connection) { const { owner, id } = parseUrl(connection.url); return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${owner}/${id}`, { + fetch(`https://api.data.world/v0/datasets/${owner}/${id}/`, { method: 'GET', headers: { 'Authorization': `Bearer ${connection.token}`, @@ -56,7 +56,7 @@ export function tables(connection) { export function schemas(connection) { return new Promise((resolve) => { query('SELECT * FROM TableColumns', connection).then((res) => { - const rows = res.rows.splice(1).map((table) => { + const rows = res.rows.map((table) => { const tableName = table[0]; const columnName = table[3]; // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer diff --git a/package.json b/package.json index e6a502aea..fc50954f5 100644 --- a/package.json +++ b/package.json @@ -168,6 +168,7 @@ "json-loader": "^0.5.4", "minimist": "^1.2.0", "mkdirp": "^0.5.1", + "nock": "^9.1.5", "node-fetch": "^1.7.2", "node-impala": "^2.0.4", "plotly.js": "^1.31.2", diff --git a/test/backend/datastores.dataworld.spec.js b/test/backend/datastores.dataworld.spec.js new file mode 100644 index 000000000..71b7ee154 --- /dev/null +++ b/test/backend/datastores.dataworld.spec.js @@ -0,0 +1,116 @@ +import nock from 'nock'; +import {assert} from 'chai'; + +import { + dataWorldConnection as connection, + dataWorldTablesResponse, + dataWorldQueryResponse, + dataWorldColumnsResponse +} from './utils.js'; +import {connect, tables, query, schemas} from '../../backend/persistent/datastores/DataWorld'; + +// Mock dataset GET request +nock('https://api.data.world/v0/datasets/falcon/test-dataset', { + reqheaders: { + 'Authorization': 'Bearer token', + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + } +}) +.get('/') +.reply(200, { + owner: 'falcon', + id: 'test-dataset', + title: 'test-dataset', + tags: [], + visibility: 'PUBLIC', + files: [] +}); + +// Mock table query POST request +nock('https://api.data.world', { + reqheaders: { + 'Authorization': 'Bearer token', + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + } +}) +.post('/v0/sql/falcon/test-dataset', 'query=SELECT%20*%20FROM%20Tables') +.query({ + includeTableSchema: 'true' +}) +.reply(200, dataWorldTablesResponse); + +// Mock query POST request +nock('https://api.data.world', { + reqheaders: { + 'Authorization': 'Bearer token', + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + } +}) +.post('/v0/sql/falcon/test-dataset', 'query=SELECT%20*%20FROM%20sampletable%20LIMIT%205') +.query({ + includeTableSchema: 'true' +}) +.reply(200, dataWorldQueryResponse); + +// Mock table columns query POST request +nock('https://api.data.world', { + reqheaders: { + 'Authorization': 'Bearer token', + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' + } +}) +.post('/v0/sql/falcon/test-dataset', 'query=SELECT%20*%20FROM%20TableColumns') +.query({ + includeTableSchema: 'true' +}) +.reply(200, dataWorldColumnsResponse); + +describe('Data World:', function () { + + it('connect succeeds', function(done) { + connect(connection).then(() => { + done(); + }).catch(done); + }); + + it('tables returns list of tables', function(done) { + tables(connection).then(result => { + assert.deepEqual(result, ['sampletable']); + done(); + }).catch(done); + }); + + it('query returns rows and column names', function(done) { + query('SELECT * FROM sampletable LIMIT 5', connection).then(result => { + assert.deepEqual( + result.columnnames, + [ 'stringcolumn', 'datecolumn', 'decimalcolumn' ] + ); + assert.deepEqual(result.rows, [ + [ 'First column', '2017-05-24', 1 ], + [ 'Second column', '2017-05-25', 2 ], + [ 'Third column', '2017-05-26', 3 ], + [ 'Fourth column', '2017-05-27', 4 ], + [ 'Fifth column', '2017-05-28', 5 ] + ]); + + done(); + }).catch(done); + }); + + it('schemas returns table schemas', function(done) { + schemas(connection).then(result => { + assert.deepEqual( + result.columnNames, + [ 'tablename', 'column_name', 'data_type' ] + ); + assert.deepEqual(result.rows, [ + [ 'sampletable', 'stringcolumn', 'string' ], + [ 'sampletable', 'datecolumn', 'date' ], + [ 'sampletable', 'decimalcolumn', 'decimal' ] + ]); + + done(); + }).catch(done); + }); +}); diff --git a/test/backend/utils.js b/test/backend/utils.js index 7c63d8552..6fb07ed17 100644 --- a/test/backend/utils.js +++ b/test/backend/utils.js @@ -296,6 +296,10 @@ export const apacheImpalaConnection = { port: 21000, database: 'plotly' }; +export const dataWorldConnection = { + url: 'https://data.world/falcon/test-dataset', + token: 'token' +}; // TODO - Add sqlite here // TODO - Add postgis in here @@ -602,3 +606,188 @@ export const apacheDrillStorage = [ } } ]; + +export const dataWorldTablesResponse = [ + { + 'fields': [ + { + 'name': 'tableId', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'tableName', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'tableTitle', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'tableDescription', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'owner', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'dataset', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + } + ] + }, + { + 'tableId': 'sampletable', + 'tableName': 'sampletable', + 'tableTitle': 'sampletable', + 'tableDescription': null, + 'owner': 'falcon', + 'dataset': 'sample-dataset' + } +]; + +export const dataWorldQueryResponse = [ + { + 'fields': [ + { + 'name': 'stringcolumn', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'datecolumn', + 'type': 'date', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#date' + }, + { + 'name': 'decimalcolumn', + 'type': 'decimal', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#decimal' + } + ] + }, + { + 'stringcolumn': 'First column', + 'datecolumn': '2017-05-24', + 'decimalcolumn': 1 + }, + { + 'stringcolumn': 'Second column', + 'datecolumn': '2017-05-25', + 'decimalcolumn': 2 + }, + { + 'stringcolumn': 'Third column', + 'datecolumn': '2017-05-26', + 'decimalcolumn': 3 + }, + { + 'stringcolumn': 'Fourth column', + 'datecolumn': '2017-05-27', + 'decimalcolumn': 4 + }, + { + 'stringcolumn': 'Fifth column', + 'datecolumn': '2017-05-28', + 'decimalcolumn': 5 + } +]; + +export const dataWorldColumnsResponse = [ + { + 'fields': [ + { + 'name': 'tableId', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'tableName', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnIndex', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnName', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnTitle', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnDescription', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnDatatype', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'columnNullable', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'owner', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + }, + { + 'name': 'dataset', + 'type': 'string', + 'rdfType': 'http://www.w3.org/2001/XMLSchema#string' + } + ] + }, + { + 'tableId': 'sampletable', + 'tableName': 'sampletable', + 'columnIndex': 1, + 'columnName': 'stringcolumn', + 'columnTitle': 'stringcolumn', + 'columnDescription': null, + 'columnDatatype': 'http://www.w3.org/2001/XMLSchema#string', + 'columnNullable': false, + 'owner': 'falcon', + 'dataset': 'test-dataset' + }, + { + 'tableId': 'sampletable', + 'tableName': 'sampletable', + 'columnIndex': 2, + 'columnName': 'datecolumn', + 'columnTitle': 'datecolumn', + 'columnDescription': null, + 'columnDatatype': 'http://www.w3.org/2001/XMLSchema#date', + 'columnNullable': false, + 'owner': 'falcon', + 'dataset': 'test-dataset' + }, + { + 'tableId': 'sampletable', + 'tableName': 'sampletable', + 'columnIndex': 3, + 'columnName': 'decimalcolumn', + 'columnTitle': 'decimalcolumn', + 'columnDescription': null, + 'columnDatatype': 'http://www.w3.org/2001/XMLSchema#decimal', + 'columnNullable': false, + 'owner': 'falcon', + 'dataset': 'test-dataset' + } +]; diff --git a/yarn.lock b/yarn.lock index a17679578..1d4cbc115 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1659,7 +1659,7 @@ chai-spies@^0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/chai-spies/-/chai-spies-0.7.1.tgz#343d99f51244212e8b17e64b93996ff7b2c2a9b1" -chai@^3.5.0: +"chai@>=1.9.2 <4.0.0", chai@^3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" dependencies: @@ -5008,7 +5008,7 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -5338,7 +5338,7 @@ lodash@4.12.0: version "4.12.0" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.12.0.tgz#2bd6dc46a040f59e686c972ed21d93dc59053258" -lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.2.1, lodash@^4.3.0, lodash@^4.6.1: +lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.2.1, lodash@^4.3.0, lodash@^4.6.1, lodash@~4.17.2: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -5830,6 +5830,20 @@ nextafter@^1.0.0: dependencies: double-bits "^1.1.0" +nock@^9.1.5: + version "9.1.5" + resolved "https://registry.yarnpkg.com/nock/-/nock-9.1.5.tgz#9e4878e0e1c050bdd93ae1e326e89461ea15618b" + dependencies: + chai ">=1.9.2 <4.0.0" + debug "^2.2.0" + deep-equal "^1.0.0" + json-stringify-safe "^5.0.1" + lodash "~4.17.2" + mkdirp "^0.5.0" + propagate "0.4.0" + qs "^6.5.1" + semver "^5.3.0" + node-abi@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.1.2.tgz#4da6caceb6685fcd31e7dd1994ef6bb7d0a9c0b2" @@ -6814,6 +6828,10 @@ prop-types@^15.0.0, prop-types@^15.5.0, prop-types@^15.5.10, prop-types@^15.5.4, loose-envify "^1.3.1" object-assign "^4.1.1" +propagate@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.4.0.tgz#f3fcca0a6fe06736a7ba572966069617c130b481" + protocol-buffers-schema@^2.0.2: version "2.2.0" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-2.2.0.tgz#d29c6cd73fb655978fb6989691180db844119f61" @@ -6874,7 +6892,7 @@ q@~1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e" -qs@^6.2.1, qs@~6.5.1: +qs@^6.2.1, qs@^6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" From 48055087df209219020465b33115162d9337548d Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 13:37:58 +0300 Subject: [PATCH 30/36] Specify type of token --- app/constants/constants.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/constants/constants.js b/app/constants/constants.js index 2e3d79abf..e233a4e88 100644 --- a/app/constants/constants.js +++ b/app/constants/constants.js @@ -189,10 +189,10 @@ export const CONNECTION_CONFIG = { 'description': 'The URL of the dataset or project on data.world' }, { - 'label': 'Token', + 'label': 'Read/Write API Token', 'value': 'token', - 'type': 'text', - 'description': 'Your data.world token. It can be obtained from https://data.world/settings/advanced' + 'type': 'password', + 'description': 'Your data.world read/write token. It can be obtained from https://data.world/settings/advanced' } ] }; From 87ff849c80f421fee85ffec312d045fd91f9416f Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 13:54:55 +0300 Subject: [PATCH 31/36] Remove unnecessary promises --- backend/persistent/datastores/DataWorld.js | 118 -------------------- backend/persistent/datastores/Datastores.js | 2 +- backend/persistent/datastores/dataworld.js | 112 +++++++++++++++++++ 3 files changed, 113 insertions(+), 119 deletions(-) delete mode 100644 backend/persistent/datastores/DataWorld.js create mode 100644 backend/persistent/datastores/dataworld.js diff --git a/backend/persistent/datastores/DataWorld.js b/backend/persistent/datastores/DataWorld.js deleted file mode 100644 index 19746de79..000000000 --- a/backend/persistent/datastores/DataWorld.js +++ /dev/null @@ -1,118 +0,0 @@ -import fetch from 'node-fetch'; -import url from 'url'; - -import Logger from '../../logger'; - -function parseUrl(datasetUrl) { - const pathnameArray = url.parse(datasetUrl).pathname.split('/'); - return { - owner: pathnameArray[1], - id: pathnameArray[2] - }; -} - -export function connect(connection) { - const { owner, id } = parseUrl(connection.url); - return new Promise((resolve, reject) => { - fetch(`https://api.data.world/v0/datasets/${owner}/${id}/`, { - method: 'GET', - headers: { - 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', - 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` - } - }) - .then(res => res.json()) - .then(json => { - // json.code is defined only when there is an error - if (json.code) { - reject(json); - } else { - resolve(); - } - }) - .catch(err => { - Logger.log(err); - throw new Error(err); - }); - }); - } - -export function tables(connection) { - return new Promise((resolve, reject) => { - query('SELECT * FROM Tables', connection).then((res) => { - const allTables = res.rows.map((table) => { - return table[0]; - }); - - resolve(allTables); - }) - .catch(err => { - reject(err); - }); - }); -} - -export function schemas(connection) { - return new Promise((resolve) => { - query('SELECT * FROM TableColumns', connection).then((res) => { - const rows = res.rows.map((table) => { - const tableName = table[0]; - const columnName = table[3]; - // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer - const columnDataType = /#(.*)/.exec(table[6])[1]; - - return [ - tableName, - columnName, - columnDataType - ]; - }); - - resolve({ - columnNames: [ 'tablename', 'column_name', 'data_type' ], - rows - }); - }) - .catch(err => { - Logger.log(err); - throw new Error(err); - }); - }); -} - -export function query(queryString, connection) { - const { owner, id } = parseUrl(connection.url); - return new Promise((resolve, reject) => { - const queryStatement = `${queryString.replace(/-/g, '_')}`; - const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryStatement)}`; - fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${connection.token}`, - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', - 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` - }, - body: params - }) - .then(res => { - return res.json(); - }) - .then(json => { - const fields = json[0].fields; - const columnnames = fields.map((field) => { - return field.name; - }); - const rows = json.slice(1).map((row) => { - return Object.values(row); - }); - resolve({ - columnnames, - rows - }); - }) - .catch(err => { - reject(err); - }); - }); -} diff --git a/backend/persistent/datastores/Datastores.js b/backend/persistent/datastores/Datastores.js index c0657a17d..17be6cad8 100644 --- a/backend/persistent/datastores/Datastores.js +++ b/backend/persistent/datastores/Datastores.js @@ -5,7 +5,7 @@ import * as ApacheDrill from './ApacheDrill'; import * as IbmDb2 from './ibmdb2'; import * as ApacheLivy from './livy'; import * as ApacheImpala from './impala'; -import * as DataWorld from './DataWorld'; +import * as DataWorld from './dataworld'; import * as DatastoreMock from './datastoremock'; /* diff --git a/backend/persistent/datastores/dataworld.js b/backend/persistent/datastores/dataworld.js new file mode 100644 index 000000000..c3c782fab --- /dev/null +++ b/backend/persistent/datastores/dataworld.js @@ -0,0 +1,112 @@ +import fetch from 'node-fetch'; +import url from 'url'; + +import Logger from '../../logger'; + +function parseUrl(datasetUrl) { + const pathnameArray = url.parse(datasetUrl).pathname.split('/'); + return { + owner: pathnameArray[1], + id: pathnameArray[2] + }; +} + +export function connect(connection) { + const { owner, id } = parseUrl(connection.url); + return fetch(`https://api.data.world/v0/datasets/${owner}/${id}/`, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` + } + }) + .then(res => res.json()) + .then(json => { + // json.code is defined only when there is an error + if (json.code) { + throw new Error(JSON.stringify(json)); + } + }) + .catch(err => { + Logger.log(err); + throw err; + }); +} + +export function tables(connection) { + return query('SELECT * FROM Tables', connection).then((res) => { + const allTables = res.rows.map((table) => { + return table[0]; + }); + + return allTables; + }) + .catch(err => { + Logger.log(err); + throw err; + }); +} + +export function schemas(connection) { + return query('SELECT * FROM TableColumns', connection).then((res) => { + const rows = res.rows.map((table) => { + const tableName = table[0]; + const columnName = table[3]; + // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer + const columnDataType = /#(.*)/.exec(table[6])[1]; + + return [ + tableName, + columnName, + columnDataType + ]; + }); + + return ({ + columnNames: [ 'tablename', 'column_name', 'data_type' ], + rows + }); + }) + .catch(err => { + Logger.log(err); + throw err; + }); +} + +export function query(queryString, connection) { + const { owner, id } = parseUrl(connection.url); + const queryStatement = `${queryString.replace(/-/g, '_')}`; + const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryStatement)}`; + + return fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${connection.token}`, + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', + 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` + }, + body: params + }) + .then(res => { + return res.json(); + }) + .then(json => { + const fields = json[0].fields; + const columnnames = fields.map((field) => { + return field.name; + }); + const rows = json.slice(1).map((row) => { + return Object.values(row); + }); + + return ({ + columnnames, + rows + }); + }) + .catch(err => { + Logger.log(err); + throw err; + }); +} From 2fbc1d58b6af10e49d390eb0dec0bf4716df24a7 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 15:35:52 +0300 Subject: [PATCH 32/36] Replace dashes in tables and schemas function --- backend/persistent/datastores/dataworld.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/persistent/datastores/dataworld.js b/backend/persistent/datastores/dataworld.js index c3c782fab..00e336c64 100644 --- a/backend/persistent/datastores/dataworld.js +++ b/backend/persistent/datastores/dataworld.js @@ -37,7 +37,7 @@ export function connect(connection) { export function tables(connection) { return query('SELECT * FROM Tables', connection).then((res) => { const allTables = res.rows.map((table) => { - return table[0]; + return table[0].replace(/-/g, '_'); }); return allTables; @@ -51,7 +51,7 @@ export function tables(connection) { export function schemas(connection) { return query('SELECT * FROM TableColumns', connection).then((res) => { const rows = res.rows.map((table) => { - const tableName = table[0]; + const tableName = table[0].replace(/-/g, '_'); const columnName = table[3]; // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer const columnDataType = /#(.*)/.exec(table[6])[1]; @@ -76,8 +76,7 @@ export function schemas(connection) { export function query(queryString, connection) { const { owner, id } = parseUrl(connection.url); - const queryStatement = `${queryString.replace(/-/g, '_')}`; - const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryStatement)}`; + const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryString)}`; return fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { method: 'POST', From 1350ec819201c270ac949a8347a40b6072d54d13 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 15:36:37 +0300 Subject: [PATCH 33/36] Return promise in connect test case --- test/backend/datastores.dataworld.spec.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/backend/datastores.dataworld.spec.js b/test/backend/datastores.dataworld.spec.js index 71b7ee154..062d7094b 100644 --- a/test/backend/datastores.dataworld.spec.js +++ b/test/backend/datastores.dataworld.spec.js @@ -67,10 +67,8 @@ nock('https://api.data.world', { describe('Data World:', function () { - it('connect succeeds', function(done) { - connect(connection).then(() => { - done(); - }).catch(done); + it('connect succeeds', function() { + return connect(connection); }); it('tables returns list of tables', function(done) { From 1ab33e8e5a4064df253920b6cc1874d659d6594c Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 20:38:38 +0300 Subject: [PATCH 34/36] Return promise in all data.world test cases --- test/backend/datastores.dataworld.spec.js | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/test/backend/datastores.dataworld.spec.js b/test/backend/datastores.dataworld.spec.js index 062d7094b..3e44b0222 100644 --- a/test/backend/datastores.dataworld.spec.js +++ b/test/backend/datastores.dataworld.spec.js @@ -71,15 +71,14 @@ describe('Data World:', function () { return connect(connection); }); - it('tables returns list of tables', function(done) { - tables(connection).then(result => { + it('tables returns list of tables', function() { + return tables(connection).then(result => { assert.deepEqual(result, ['sampletable']); - done(); - }).catch(done); + }); }); - it('query returns rows and column names', function(done) { - query('SELECT * FROM sampletable LIMIT 5', connection).then(result => { + it('query returns rows and column names', function() { + return query('SELECT * FROM sampletable LIMIT 5', connection).then(result => { assert.deepEqual( result.columnnames, [ 'stringcolumn', 'datecolumn', 'decimalcolumn' ] @@ -91,13 +90,11 @@ describe('Data World:', function () { [ 'Fourth column', '2017-05-27', 4 ], [ 'Fifth column', '2017-05-28', 5 ] ]); - - done(); - }).catch(done); + }); }); - it('schemas returns table schemas', function(done) { - schemas(connection).then(result => { + it('schemas returns table schemas', function() { + return schemas(connection).then(result => { assert.deepEqual( result.columnNames, [ 'tablename', 'column_name', 'data_type' ] @@ -107,8 +104,6 @@ describe('Data World:', function () { [ 'sampletable', 'datecolumn', 'date' ], [ 'sampletable', 'decimalcolumn', 'decimal' ] ]); - - done(); - }).catch(done); + }); }); }); From 04ad51a1408e8972461d09ce792bb96cd2a7d7a4 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Tue, 9 Jan 2018 20:39:31 +0300 Subject: [PATCH 35/36] Add data.world test script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index fc50954f5..3bffdbd17 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test-unit-all-watch": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --bail --full-trace --timeout 90000 --compilers js:babel-register --recursive test/**/*.spec.js --watch", "test-unit-watch": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --bail --full-trace --timeout 90000 --watch --compilers js:babel-register ", "test-unit-certificates": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/certificates.spec.js", + "test-unit-dataworld": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.dataworld.spec.js", "test-unit-ibmdb": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.ibmdb.spec.js", "test-unit-impala": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.impala.spec.js", "test-unit-livy": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --full-trace --timeout 90000 --compilers js:babel-register test/backend/datastores.livy.spec.js", From de82e14bc108dda9f0dac33ef45ee930f512c0d0 Mon Sep 17 00:00:00 2001 From: Kinuthia Ndung'u Date: Wed, 10 Jan 2018 07:14:01 +0300 Subject: [PATCH 36/36] Update filename --- test/backend/datastores.dataworld.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backend/datastores.dataworld.spec.js b/test/backend/datastores.dataworld.spec.js index 3e44b0222..b21932be0 100644 --- a/test/backend/datastores.dataworld.spec.js +++ b/test/backend/datastores.dataworld.spec.js @@ -7,7 +7,7 @@ import { dataWorldQueryResponse, dataWorldColumnsResponse } from './utils.js'; -import {connect, tables, query, schemas} from '../../backend/persistent/datastores/DataWorld'; +import {connect, tables, query, schemas} from '../../backend/persistent/datastores/dataworld'; // Mock dataset GET request nock('https://api.data.world/v0/datasets/falcon/test-dataset', {