Skip to main content
Solved

RAW Tables - retrieve_dataframe failing


Forum|alt.badge.img+2

I have been working on this for some time now. It was all running fine till now but suddenly it is failing now, and it is giving an error for this function. Is there something that has been changed in the platform/library?

DB name and table name are all accurate and was running fine till today. 

 

lp_df_input = client.raw.rows.retrieve_dataframe(db_name="Eashwar_MOTDB-db",table_name="lp_input",limit=None,columns=None)

Error: 

---------------------------------------------------------------------------
MissingSchema                             Traceback (most recent call last)
Cell In[30], line 1
----> 1 lp_df_input = client.raw.rows.retrieve_dataframe(db_name="Eashwar_MOTDB-db",table_name="lp_input",limit=None,columns=None)

File /lib/python3.11/site-packages/cognite/client/_api/raw.py:613, in RawRowsAPI.retrieve_dataframe(self, db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
    589 """`Retrieve rows in a table as a pandas dataframe. <https://developer.cognite.com/api#tag/Raw/operation/getRows>`_
    590 
    591 Rowkeys are used as the index.
   (...)
    610         >>> df = c.raw.rows.retrieve_dataframe("db1", "t1", limit=5)
    611 """
    612 pd = cast(Any, local_import("pandas"))
--> 613 rows = self.list(db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
    614 idx = [r.key for r in rows]
    615 cols = [r.columns for r in rows]

File /lib/python3.11/site-packages/cognite/client/_api/raw.py:537, in RawRowsAPI.list(self, db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
    501 """`List rows in a table. <https://developer.cognite.com/api#tag/Raw/operation/getRows>`_
    502 
    503 Args:
   (...)
    534         ...     row_list # do something with the rows
    535 """
    536 if is_unlimited(limit):
--> 537     cursors = self._get(
    538         url_path=utils._auxiliary.interpolate_and_url_encode(
    539             "/raw/dbs/{}/tables/{}/cursors", db_name, table_name
    540         ),
    541         params={
    542             "minLastUpdatedTime": min_last_updated_time,
    543             "maxLastUpdatedTime": max_last_updated_time,
    544             "numberOfCursors": self._config.max_workers,
    545         },
    546     ).json()["items"]
    547 else:
    548     cursors = [None]

File /lib/python3.11/site-packages/cognite/client/_api_client.py:134, in APIClient._get(self, url_path, params, headers)
    131 def _get(
    132     self, url_path: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, Any]] = None
    133 ) -> Response:
--> 134     return self._do_request("GET", url_path, params=params, headers=headers, timeout=self._config.timeout)

File /lib/python3.11/site-packages/cognite/client/_api_client.py:185, in APIClient._do_request(self, method, url_path, accept, **kwargs)
    182 kwargs.setdefault("allow_redirects", False)
    184 if is_retryable:
--> 185     res = self._http_client_with_retry.request(method=method, url=full_url, **kwargs)
    186 else:
    187     res = self._http_client.request(method=method, url=full_url, **kwargs)

File /lib/python3.11/site-packages/cognite/client/_http_client.py:113, in HTTPClient.request(self, method, url, **kwargs)
    111 while True:
    112     try:
--> 113         res = self._do_request(method=method, url=url, **kwargs)
    114         last_status = res.status_code
    115         retry_tracker.status += 1

File /lib/python3.11/site-packages/cognite/client/_http_client.py:162, in HTTPClient._do_request(self, method, url, **kwargs)
    160         raise CogniteConnectionRefused from e
    161     raise CogniteConnectionError from e
--> 162 raise e

File /lib/python3.11/site-packages/cognite/client/_http_client.py:143, in HTTPClient._do_request(self, method, url, **kwargs)
    136 """requests/urllib3 adds 2 or 3 layers of exceptions on top of built-in networking exceptions.
    137 
    138 Sometimes the appropriate built-in networking exception is not in the context, sometimes the requests
    139 exception is not in the context, so we need to check for the appropriate built-in exceptions,
    140 urllib3 exceptions, and requests exceptions.
    141 """
    142 try:
--> 143     res = self.session.request(method=method, url=url, **kwargs)
    144     return res
    145 except Exception as e:

File /lib/python3.11/site-packages/requests/sessions.py:575, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    562 # Create the Request.
    563 req = Request(
    564     method=method.upper(),
    565     url=url,
   (...)
    573     hooks=hooks,
    574 )
--> 575 prep = self.prepare_request(req)
    577 proxies = proxies or {}
    579 settings = self.merge_environment_settings(
    580     prep.url, proxies, stream, verify, cert
    581 )

File /lib/python3.11/site-packages/requests/sessions.py:486, in Session.prepare_request(self, request)
    483     auth = get_netrc_auth(request.url)
    485 p = PreparedRequest()
--> 486 p.prepare(
    487     method=request.method.upper(),
    488     url=request.url,
    489     files=request.files,
    490     data=request.data,
    491     json=request.json,
    492     headers=merge_setting(
    493         request.headers, self.headers, dict_class=CaseInsensitiveDict
    494     ),
    495     params=merge_setting(request.params, self.params),
    496     auth=merge_setting(auth, self.auth),
    497     cookies=merged_cookies,
    498     hooks=merge_hooks(request.hooks, self.hooks),
    499 )
    500 return p

File /lib/python3.11/site-packages/requests/models.py:368, in PreparedRequest.prepare(self, method, url, headers, files, data, params, auth, cookies, hooks, json)
    365 """Prepares the entire request with the given parameters."""
    367 self.prepare_method(method)
--> 368 self.prepare_url(url, params)
    369 self.prepare_headers(headers)
    370 self.prepare_cookies(cookies)

File /lib/python3.11/site-packages/requests/models.py:439, in PreparedRequest.prepare_url(self, url, params)
    436     raise InvalidURL(*e.args)
    438 if not scheme:
--> 439     raise MissingSchema(
    440         f"Invalid URL {url!r}: No scheme supplied. "
    441         f"Perhaps you meant https://{url}?"
    442     )
    444 if not host:
    445     raise InvalidURL(f"Invalid URL {url!r}: No host supplied")

MissingSchema: Invalid URL '/api/v1/projects/undefined/raw/dbs/Eashwar_MOTDB-db/tables/lp_input/cursors': No scheme supplied. Perhaps you meant https:///api/v1/projects/undefined/raw/dbs/Eashwar_MOTDB-db/tables/lp_input/cursors?
​

 

Best answer by Håkon V. Treider

From your posted error message, the CDF project settings still appears wrong (it says undefined, while I would expect it to be something like company-prod or company-dev). Please verify your connection details. For instance, if you read some parameters from a config file or an environment variable, double-check the value of these.

View original
Did this topic help you find an answer to your question?

5 replies

Forum|alt.badge.img

I am unable to reproduce this error, could you please run the following:

print(client.version)

 

From the “invalid URL” in the error message I can see that the Python SDK thinks your CDF project = “undefined”. Could you please verify your connection details? For instance by checking token/inspect endpoint:

my_projects = client.iam.token.inspect().projects
print([p.url_name for p in my_projects])

 


Forum|alt.badge.img+2
  • Author
  • 42 replies
  • August 10, 2023

Hi @Håkon V. Treider 

print(client.version)

6.13.1

my_projects = client.iam.token.inspect().projects
print([p.url_name for p in my_projects])
---------------------------------------------------------------------------
MissingSchema                             Traceback (most recent call last)
Cell In[33], line 1
----> 1 my_projects = client.iam.token.inspect().projects 
      2 print([p.url_name for p in my_projects]) 

File /lib/python3.11/site-packages/cognite/client/_api/iam.py:184, in TokenAPI.inspect(self)
    168 def inspect(self) -> TokenInspection:
    169     """Inspect a token.
    170 
    171     Get details about which projects it belongs to and which capabilities are granted to it.
   (...)
    182             >>> res = c.iam.token.inspect()
    183     """
--> 184     return TokenInspection._load(self._get("/api/v1/token/inspect").json())

File /lib/python3.11/site-packages/cognite/client/_api_client.py:134, in APIClient._get(self, url_path, params, headers)
    131 def _get(
    132     self, url_path: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, Any]] = None
    133 ) -> Response:
--> 134     return self._do_request("GET", url_path, params=params, headers=headers, timeout=self._config.timeout)

File /lib/python3.11/site-packages/cognite/client/_api_client.py:185, in APIClient._do_request(self, method, url_path, accept, **kwargs)
    182 kwargs.setdefault("allow_redirects", False)
    184 if is_retryable:
--> 185     res = self._http_client_with_retry.request(method=method, url=full_url, **kwargs)
    186 else:
    187     res = self._http_client.request(method=method, url=full_url, **kwargs)

File /lib/python3.11/site-packages/cognite/client/_http_client.py:113, in HTTPClient.request(self, method, url, **kwargs)
    111 while True:
    112     try:
--> 113         res = self._do_request(method=method, url=url, **kwargs)
    114         last_status = res.status_code
    115         retry_tracker.status += 1

File /lib/python3.11/site-packages/cognite/client/_http_client.py:162, in HTTPClient._do_request(self, method, url, **kwargs)
    160         raise CogniteConnectionRefused from e
    161     raise CogniteConnectionError from e
--> 162 raise e

File /lib/python3.11/site-packages/cognite/client/_http_client.py:143, in HTTPClient._do_request(self, method, url, **kwargs)
    136 """requests/urllib3 adds 2 or 3 layers of exceptions on top of built-in networking exceptions.
    137 
    138 Sometimes the appropriate built-in networking exception is not in the context, sometimes the requests
    139 exception is not in the context, so we need to check for the appropriate built-in exceptions,
    140 urllib3 exceptions, and requests exceptions.
    141 """
    142 try:
--> 143     res = self.session.request(method=method, url=url, **kwargs)
    144     return res
    145 except Exception as e:

File /lib/python3.11/site-packages/requests/sessions.py:575, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    562 # Create the Request.
    563 req = Request(
    564     method=method.upper(),
    565     url=url,
   (...)
    573     hooks=hooks,
    574 )
--> 575 prep = self.prepare_request(req)
    577 proxies = proxies or {}
    579 settings = self.merge_environment_settings(
    580     prep.url, proxies, stream, verify, cert
    581 )

File /lib/python3.11/site-packages/requests/sessions.py:486, in Session.prepare_request(self, request)
    483     auth = get_netrc_auth(request.url)
    485 p = PreparedRequest()
--> 486 p.prepare(
    487     method=request.method.upper(),
    488     url=request.url,
    489     files=request.files,
    490     data=request.data,
    491     json=request.json,
    492     headers=merge_setting(
    493         request.headers, self.headers, dict_class=CaseInsensitiveDict
    494     ),
    495     params=merge_setting(request.params, self.params),
    496     auth=merge_setting(auth, self.auth),
    497     cookies=merged_cookies,
    498     hooks=merge_hooks(request.hooks, self.hooks),
    499 )
    500 return p

File /lib/python3.11/site-packages/requests/models.py:368, in PreparedRequest.prepare(self, method, url, headers, files, data, params, auth, cookies, hooks, json)
    365 """Prepares the entire request with the given parameters."""
    367 self.prepare_method(method)
--> 368 self.prepare_url(url, params)
    369 self.prepare_headers(headers)
    370 self.prepare_cookies(cookies)

File /lib/python3.11/site-packages/requests/models.py:439, in PreparedRequest.prepare_url(self, url, params)
    436     raise InvalidURL(*e.args)
    438 if not scheme:
--> 439     raise MissingSchema(
    440         f"Invalid URL {url!r}: No scheme supplied. "
    441         f"Perhaps you meant https://{url}?"
    442     )
    444 if not host:
    445     raise InvalidURL(f"Invalid URL {url!r}: No host supplied")

MissingSchema: Invalid URL 'undefined/api/v1/token/inspect': No scheme supplied. Perhaps you meant https://undefined/api/v1/token/inspect?

 


Forum|alt.badge.img

From your posted error message, the CDF project settings still appears wrong (it says undefined, while I would expect it to be something like company-prod or company-dev). Please verify your connection details. For instance, if you read some parameters from a config file or an environment variable, double-check the value of these.


Forum|alt.badge.img+2
  • Author
  • 42 replies
  • August 12, 2023

Hi @Håkon V. Treider 

It automatically got resolved now. Now it is not throwing any error. 


  • Seasoned Practitioner
  • 223 replies
  • August 14, 2023

Hi @eashwar11, great to hear that your issue is sorted. Would you mind marking the best answer? 


Reply


Cookie Policy

We use cookies to enhance and personalize your experience. If you accept you agree to our full cookie policy. Learn more about our cookies.

 
Cookie Settings