This repository was archived by the owner on Aug 29, 2025. It is now read-only.
DB Connector for data.world #332
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| import fetch from 'node-fetch'; | ||
| import url from 'url'; | ||
|
|
||
| import Logger from '../../logger'; | ||
|
|
||
| function parseUrl(datasetUrl) { | ||
| const pathnameArray = url.parse(datasetUrl).pathname.split('/'); | ||
| return { | ||
| owner: pathnameArray[1], | ||
| id: pathnameArray[2] | ||
| }; | ||
| } | ||
|
|
||
| export function connect(connection) { | ||
| const { owner, id } = parseUrl(connection.url); | ||
| return fetch(`https://api.data.world/v0/datasets/${owner}/${id}/`, { | ||
| method: 'GET', | ||
| headers: { | ||
| 'Authorization': `Bearer ${connection.token}`, | ||
| 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', | ||
| 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` | ||
| } | ||
| }) | ||
| .then(res => res.json()) | ||
| .then(json => { | ||
| // json.code is defined only when there is an error | ||
| if (json.code) { | ||
| throw new Error(JSON.stringify(json)); | ||
| } | ||
| }) | ||
| .catch(err => { | ||
| Logger.log(err); | ||
| throw err; | ||
| }); | ||
| } | ||
|
|
||
| export function tables(connection) { | ||
| return query('SELECT * FROM Tables', connection).then((res) => { | ||
| const allTables = res.rows.map((table) => { | ||
| return table[0].replace(/-/g, '_'); | ||
| }); | ||
|
|
||
| return allTables; | ||
| }) | ||
| .catch(err => { | ||
| Logger.log(err); | ||
| throw err; | ||
| }); | ||
| } | ||
|
|
||
| export function schemas(connection) { | ||
| return query('SELECT * FROM TableColumns', connection).then((res) => { | ||
| const rows = res.rows.map((table) => { | ||
| const tableName = table[0].replace(/-/g, '_'); | ||
| const columnName = table[3]; | ||
| // Extract the datatype from datatype url e.g. http://www.w3.org/2001/XMLSchema#integer | ||
| const columnDataType = /#(.*)/.exec(table[6])[1]; | ||
|
|
||
| return [ | ||
| tableName, | ||
| columnName, | ||
| columnDataType | ||
| ]; | ||
| }); | ||
|
|
||
| return ({ | ||
| columnNames: [ 'tablename', 'column_name', 'data_type' ], | ||
| rows | ||
| }); | ||
| }) | ||
| .catch(err => { | ||
| Logger.log(err); | ||
| throw err; | ||
| }); | ||
| } | ||
|
|
||
| export function query(queryString, connection) { | ||
| const { owner, id } = parseUrl(connection.url); | ||
| const params = `${encodeURIComponent('query')}=${encodeURIComponent(queryString)}`; | ||
|
|
||
| return fetch(`https://api.data.world/v0/sql/${owner}/${id}?includeTableSchema=true`, { | ||
| method: 'POST', | ||
| headers: { | ||
| 'Authorization': `Bearer ${connection.token}`, | ||
| 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8', | ||
| 'User-Agent': `Falcon/Plotly - ${process.env.npm_package_version}` | ||
| }, | ||
| body: params | ||
| }) | ||
| .then(res => { | ||
| return res.json(); | ||
| }) | ||
| .then(json => { | ||
| const fields = json[0].fields; | ||
| const columnnames = fields.map((field) => { | ||
| return field.name; | ||
| }); | ||
| const rows = json.slice(1).map((row) => { | ||
| return Object.values(row); | ||
| }); | ||
|
|
||
| return ({ | ||
| columnnames, | ||
| rows | ||
| }); | ||
| }) | ||
| .catch(err => { | ||
| Logger.log(err); | ||
| throw err; | ||
| }); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'd like to change this. Instead of having the user to input an URL and the connector to parse the URL to determine
ownerand datasetid. I'd like the user to inputownerandid:getPathNames)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hi, @n-riesco, unless you feel strongly about this, we would like to use the URL. That would be consistent with our other connectors work (e.g. Tableau and Google Data Studio) and is more usable, as users only have one value to copy and paste, which is readily available on the browser for the dataset they are looking at. We do a little bit of extra work in code to spare the user in real life. Also, using URIs that contain information to parse out isn't without a precedent, the most common is JDBC.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No, I don't feel strongly about it.
@tarzzz @chriddyp are you OK with using an URL here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This sounds like a good reason to me 👍