Solved

RAW Tables - retrieve_dataframe failing

  • 10 August 2023
  • 5 replies
  • 60 views

Badge +2

I have been working on this for some time now. It was all running fine till now but suddenly it is failing now, and it is giving an error for this function. Is there something that has been changed in the platform/library?

DB name and table name are all accurate and was running fine till today. 

 

lp_df_input = client.raw.rows.retrieve_dataframe(db_name="Eashwar_MOTDB-db",table_name="lp_input",limit=None,columns=None)

Error: 

---------------------------------------------------------------------------
MissingSchema Traceback (most recent call last)
Cell In[30], line 1
----> 1 lp_df_input = client.raw.rows.retrieve_dataframe(db_name="Eashwar_MOTDB-db",table_name="lp_input",limit=None,columns=None)

File /lib/python3.11/site-packages/cognite/client/_api/raw.py:613, in RawRowsAPI.retrieve_dataframe(self, db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
589 """`Retrieve rows in a table as a pandas dataframe. <https://developer.cognite.com/api#tag/Raw/operation/getRows>`_
590
591 Rowkeys are used as the index.
(...)
610 >>> df = c.raw.rows.retrieve_dataframe("db1", "t1", limit=5)
611 """
612 pd = cast(Any, local_import("pandas"))
--> 613 rows = self.list(db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
614 idx = [r.key for r in rows]
615 cols = [r.columns for r in rows]

File /lib/python3.11/site-packages/cognite/client/_api/raw.py:537, in RawRowsAPI.list(self, db_name, table_name, min_last_updated_time, max_last_updated_time, columns, limit)
501 """`List rows in a table. <https://developer.cognite.com/api#tag/Raw/operation/getRows>`_
502
503 Args:
(...)
534 ... row_list # do something with the rows
535 """
536 if is_unlimited(limit):
--> 537 cursors = self._get(
538 url_path=utils._auxiliary.interpolate_and_url_encode(
539 "/raw/dbs/{}/tables/{}/cursors", db_name, table_name
540 ),
541 params={
542 "minLastUpdatedTime": min_last_updated_time,
543 "maxLastUpdatedTime": max_last_updated_time,
544 "numberOfCursors": self._config.max_workers,
545 },
546 ).json()["items"]
547 else:
548 cursors = [None]

File /lib/python3.11/site-packages/cognite/client/_api_client.py:134, in APIClient._get(self, url_path, params, headers)
131 def _get(
132 self, url_path: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, Any]] = None
133 ) -> Response:
--> 134 return self._do_request("GET", url_path, params=params, headers=headers, timeout=self._config.timeout)

File /lib/python3.11/site-packages/cognite/client/_api_client.py:185, in APIClient._do_request(self, method, url_path, accept, **kwargs)
182 kwargs.setdefault("allow_redirects", False)
184 if is_retryable:
--> 185 res = self._http_client_with_retry.request(method=method, url=full_url, **kwargs)
186 else:
187 res = self._http_client.request(method=method, url=full_url, **kwargs)

File /lib/python3.11/site-packages/cognite/client/_http_client.py:113, in HTTPClient.request(self, method, url, **kwargs)
111 while True:
112 try:
--> 113 res = self._do_request(method=method, url=url, **kwargs)
114 last_status = res.status_code
115 retry_tracker.status += 1

File /lib/python3.11/site-packages/cognite/client/_http_client.py:162, in HTTPClient._do_request(self, method, url, **kwargs)
160 raise CogniteConnectionRefused from e
161 raise CogniteConnectionError from e
--> 162 raise e

File /lib/python3.11/site-packages/cognite/client/_http_client.py:143, in HTTPClient._do_request(self, method, url, **kwargs)
136 """requests/urllib3 adds 2 or 3 layers of exceptions on top of built-in networking exceptions.
137
138 Sometimes the appropriate built-in networking exception is not in the context, sometimes the requests
139 exception is not in the context, so we need to check for the appropriate built-in exceptions,
140 urllib3 exceptions, and requests exceptions.
141 """
142 try:
--> 143 res = self.session.request(method=method, url=url, **kwargs)
144 return res
145 except Exception as e:

File /lib/python3.11/site-packages/requests/sessions.py:575, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
562 # Create the Request.
563 req = Request(
564 method=method.upper(),
565 url=url,
(...)
573 hooks=hooks,
574 )
--> 575 prep = self.prepare_request(req)
577 proxies = proxies or {}
579 settings = self.merge_environment_settings(
580 prep.url, proxies, stream, verify, cert
581 )

File /lib/python3.11/site-packages/requests/sessions.py:486, in Session.prepare_request(self, request)
483 auth = get_netrc_auth(request.url)
485 p = PreparedRequest()
--> 486 p.prepare(
487 method=request.method.upper(),
488 url=request.url,
489 files=request.files,
490 data=request.data,
491 json=request.json,
492 headers=merge_setting(
493 request.headers, self.headers, dict_class=CaseInsensitiveDict
494 ),
495 params=merge_setting(request.params, self.params),
496 auth=merge_setting(auth, self.auth),
497 cookies=merged_cookies,
498 hooks=merge_hooks(request.hooks, self.hooks),
499 )
500 return p

File /lib/python3.11/site-packages/requests/models.py:368, in PreparedRequest.prepare(self, method, url, headers, files, data, params, auth, cookies, hooks, json)
365 """Prepares the entire request with the given parameters."""
367 self.prepare_method(method)
--> 368 self.prepare_url(url, params)
369 self.prepare_headers(headers)
370 self.prepare_cookies(cookies)

File /lib/python3.11/site-packages/requests/models.py:439, in PreparedRequest.prepare_url(self, url, params)
436 raise InvalidURL(*e.args)
438 if not scheme:
--> 439 raise MissingSchema(
440 f"Invalid URL {url!r}: No scheme supplied. "
441 f"Perhaps you meant https://{url}?"
442 )
444 if not host:
445 raise InvalidURL(f"Invalid URL {url!r}: No host supplied")

MissingSchema: Invalid URL '/api/v1/projects/undefined/raw/dbs/Eashwar_MOTDB-db/tables/lp_input/cursors': No scheme supplied. Perhaps you meant https:///api/v1/projects/undefined/raw/dbs/Eashwar_MOTDB-db/tables/lp_input/cursors?


 

icon

Best answer by Håkon V. Treider 11 August 2023, 13:58

View original

5 replies

Userlevel 4
Badge

I am unable to reproduce this error, could you please run the following:

print(client.version)

 

From the “invalid URL” in the error message I can see that the Python SDK thinks your CDF project = “undefined”. Could you please verify your connection details? For instance by checking token/inspect endpoint:

my_projects = client.iam.token.inspect().projects
print([p.url_name for p in my_projects])

 

Badge +2

Hi @Håkon V. Treider 

print(client.version)

6.13.1

my_projects = client.iam.token.inspect().projects
print([p.url_name for p in my_projects])
---------------------------------------------------------------------------
MissingSchema Traceback (most recent call last)
Cell In[33], line 1
----> 1 my_projects = client.iam.token.inspect().projects
2 print([p.url_name for p in my_projects])

File /lib/python3.11/site-packages/cognite/client/_api/iam.py:184, in TokenAPI.inspect(self)
168 def inspect(self) -> TokenInspection:
169 """Inspect a token.
170
171 Get details about which projects it belongs to and which capabilities are granted to it.
(...)
182 >>> res = c.iam.token.inspect()
183 """
--> 184 return TokenInspection._load(self._get("/api/v1/token/inspect").json())

File /lib/python3.11/site-packages/cognite/client/_api_client.py:134, in APIClient._get(self, url_path, params, headers)
131 def _get(
132 self, url_path: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, Any]] = None
133 ) -> Response:
--> 134 return self._do_request("GET", url_path, params=params, headers=headers, timeout=self._config.timeout)

File /lib/python3.11/site-packages/cognite/client/_api_client.py:185, in APIClient._do_request(self, method, url_path, accept, **kwargs)
182 kwargs.setdefault("allow_redirects", False)
184 if is_retryable:
--> 185 res = self._http_client_with_retry.request(method=method, url=full_url, **kwargs)
186 else:
187 res = self._http_client.request(method=method, url=full_url, **kwargs)

File /lib/python3.11/site-packages/cognite/client/_http_client.py:113, in HTTPClient.request(self, method, url, **kwargs)
111 while True:
112 try:
--> 113 res = self._do_request(method=method, url=url, **kwargs)
114 last_status = res.status_code
115 retry_tracker.status += 1

File /lib/python3.11/site-packages/cognite/client/_http_client.py:162, in HTTPClient._do_request(self, method, url, **kwargs)
160 raise CogniteConnectionRefused from e
161 raise CogniteConnectionError from e
--> 162 raise e

File /lib/python3.11/site-packages/cognite/client/_http_client.py:143, in HTTPClient._do_request(self, method, url, **kwargs)
136 """requests/urllib3 adds 2 or 3 layers of exceptions on top of built-in networking exceptions.
137
138 Sometimes the appropriate built-in networking exception is not in the context, sometimes the requests
139 exception is not in the context, so we need to check for the appropriate built-in exceptions,
140 urllib3 exceptions, and requests exceptions.
141 """
142 try:
--> 143 res = self.session.request(method=method, url=url, **kwargs)
144 return res
145 except Exception as e:

File /lib/python3.11/site-packages/requests/sessions.py:575, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
562 # Create the Request.
563 req = Request(
564 method=method.upper(),
565 url=url,
(...)
573 hooks=hooks,
574 )
--> 575 prep = self.prepare_request(req)
577 proxies = proxies or {}
579 settings = self.merge_environment_settings(
580 prep.url, proxies, stream, verify, cert
581 )

File /lib/python3.11/site-packages/requests/sessions.py:486, in Session.prepare_request(self, request)
483 auth = get_netrc_auth(request.url)
485 p = PreparedRequest()
--> 486 p.prepare(
487 method=request.method.upper(),
488 url=request.url,
489 files=request.files,
490 data=request.data,
491 json=request.json,
492 headers=merge_setting(
493 request.headers, self.headers, dict_class=CaseInsensitiveDict
494 ),
495 params=merge_setting(request.params, self.params),
496 auth=merge_setting(auth, self.auth),
497 cookies=merged_cookies,
498 hooks=merge_hooks(request.hooks, self.hooks),
499 )
500 return p

File /lib/python3.11/site-packages/requests/models.py:368, in PreparedRequest.prepare(self, method, url, headers, files, data, params, auth, cookies, hooks, json)
365 """Prepares the entire request with the given parameters."""
367 self.prepare_method(method)
--> 368 self.prepare_url(url, params)
369 self.prepare_headers(headers)
370 self.prepare_cookies(cookies)

File /lib/python3.11/site-packages/requests/models.py:439, in PreparedRequest.prepare_url(self, url, params)
436 raise InvalidURL(*e.args)
438 if not scheme:
--> 439 raise MissingSchema(
440 f"Invalid URL {url!r}: No scheme supplied. "
441 f"Perhaps you meant https://{url}?"
442 )
444 if not host:
445 raise InvalidURL(f"Invalid URL {url!r}: No host supplied")

MissingSchema: Invalid URL 'undefined/api/v1/token/inspect': No scheme supplied. Perhaps you meant https://undefined/api/v1/token/inspect?

 

Userlevel 4
Badge

From your posted error message, the CDF project settings still appears wrong (it says undefined, while I would expect it to be something like company-prod or company-dev). Please verify your connection details. For instance, if you read some parameters from a config file or an environment variable, double-check the value of these.

Badge +2

Hi @Håkon V. Treider 

It automatically got resolved now. Now it is not throwing any error. 

Userlevel 3

Hi @eashwar11, great to hear that your issue is sorted. Would you mind marking the best answer? 

Reply