Port forwarding not working with python requests library

[Rami-Kassouf-FOO]

Which project are you reporting a bug for?

kr8s

What happened?

kubectl create namespace argo
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

after the apply you will get an initial root token, put this token as the password in the payload and it should be reproduced

This works (with manual port-forwarding)

import requests
from utils.kubernetes_utils import K8

url = "http://127.0.0.1:1009/api/v1/session"
payload = {"username": "admin", "password": "---"}
headers = {"Content-Type": "application/json"}

response = requests.post(url, json=payload, headers=headers, verify=False)

print(response.status_code, response.text)

This does not work

import requests
from utils.kubernetes_utils import K8

payload = {"username": "admin", "password": "---"}
headers = {"Content-Type": "application/json"}

with K8().get_service("argocd-server", namespace="argo").portforward(remote_port=80, local_port="auto") as local_port:
    url = f"http://127.0.0.1:{local_port}/api/v1/session"
    response = requests.post(url, json=payload, headers=headers, verify=False)
    print(response.status_code, response.text)

Anything else?

Error Log

Traceback (most recent call last):
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connectionpool.py", line 789, in urlopen
    response = self._make_request(
               ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connectionpool.py", line 536, in _make_request
    response = conn.getresponse()
               ^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connection.py", line 507, in getresponse
    httplib_response = super().getresponse()
                       ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 1395, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 325, in begin
    version, status, reason = self._read_status()
                              ^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 294, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\adapters.py", line 667, in send
    resp = conn.urlopen(
           ^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connectionpool.py", line 843, in urlopen       
    retries = retries.increment(
              ^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\util\retry.py", line 474, in increment
    raise reraise(type(error), error, _stacktrace)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\util\util.py", line 38, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connectionpool.py", line 789, in urlopen       
    response = self._make_request(
               ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connectionpool.py", line 536, in _make_request 
    response = conn.getresponse()
               ^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\urllib3\connection.py", line 507, in getresponse       
    httplib_response = super().getresponse()
                       ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 1395, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 325, in begin
    version, status, reason = self._read_status()
                              ^^^^^^^^^^^^^^^^^^^
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.11_3.11.2544.0_x64__qbz5n2kfra8p0\Lib\http\client.py", line 294, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\Users\User\Desktop\FOO_\Project Setup Automation\test.py", line 38, in <module>
    response = requests.post(url, json=payload, headers=headers, verify=False)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\api.py", line 115, in post
    return request("post", url, data=data, json=json, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\api.py", line 59, in request
    return session.request(method=method, url=url, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\sessions.py", line 589, in request
    resp = self.send(prep, **send_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\sessions.py", line 703, in send
    r = adapter.send(request, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\User\Desktop\FOO_\Project Setup Automation\.venv\Lib\site-packages\requests\adapters.py", line 682, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

[Rami-Kassouf-FOO]

From the additional info I have gathered :

• The error still happens with kr8s v 0.19.1.
• Connecting to HASHICORP vault with portforwarding works as intended.
• The error happens here when the switch protocols happens:

2025-02-03 10:09:27,316 - httpx - INFO - HTTP Request: GET https://AWS_IP/api/v1/namespaces/argo/pods/argocd-server-888b5d99f-zq4mg "HTTP/1.1 200 OK"
2025-02-03 10:09:27,317 - httpcore.http11 - DEBUG - receive_response_body.started request=<Request [b'GET']>
2025-02-03 10:09:27,318 - httpcore.http11 - DEBUG - receive_response_body.complete
2025-02-03 10:09:27,318 - httpcore.http11 - DEBUG - response_closed.started
2025-02-03 10:09:27,318 - httpcore.http11 - DEBUG - response_closed.complete
2025-02-03 10:09:27,333 - httpcore.http11 - DEBUG - send_request_headers.started request=<Request [b'GET']>
2025-02-03 10:09:27,333 - httpcore.http11 - DEBUG - send_request_headers.complete
2025-02-03 10:09:27,333 - httpcore.http11 - DEBUG - send_request_body.started request=<Request [b'GET']>
2025-02-03 10:09:27,333 - httpcore.http11 - DEBUG - send_request_body.complete
2025-02-03 10:09:27,333 - httpcore.http11 - DEBUG - receive_response_headers.started request=<Request [b'GET']>
2025-02-03 10:09:27,450 - httpcore.http11 - DEBUG - receive_response_headers.complete return_value=(b'HTTP/1.1', 101, b'Switching Protocols', [(b'Upgrade', b'websocket'), (b'Connection', b'Upgrade'), (b'Sec-WebSocket-Accept', b'GHY385CBC70ha/yn35vdCKJtIwo='), (b'Sec-WebSocket-Protocol', b'')])
2025-02-03 10:09:27,450 - httpx - INFO - HTTP Request: GET https://AWS_IP/api/v1/namespaces/argo/pods/argocd-server-888b5d99f-zq4mg/portforward?name=argocd-server-888b5d99f-zq4mg&namespace=argo&ports=80&_preload_content=false "HTTP/1.1 101 Switching Protocols"
2025-02-03 10:09:27,453 - httpcore.http11 - DEBUG - response_closed.started
2025-02-03 10:09:27,453 - httpcore.http11 - DEBUG - response_closed.complete

[jacobtomlinson]

I tried a simple example by starting an nginx web server Pod and getting the homepage with a port forward and requests.

This works as expected:

# Create web server Pod
kubectl run webserver --image nginx --port 80
kubectl expose po webserver

import kr8s
import requests

[svc] = kr8s.get("svc", "webserver")
with svc.portforward(remote_port=80, local_port="auto") as local_port:
    resp = requests.get(f"http://localhost:{local_port}/")
    print(resp.text)
# <!DOCTYPE html>\n<html>\n<head>\n<title>Welcome to nginx!</title>...

# Clean up
kubectl delete svc/webserver po/webserver

Given that you are also able to access Vault via a port forward it sounds like there is something specific to the Argo request you are making. The error messages you shared show the remote server not responding and requests timing out. This could be due to the port forward not passing data back and forth, but it's unclear why this is happening.

The logs you shared up to the Switching Protocols line are as expected, the port forward uses websockets and the last thing it does before the data exchange starts is switch protocols. However, after that line the connection closes which suggests the argo server closes the connection.

To debug this further we need a small example like the one I shared above that reproduces the issue. That way I can attach a debugger and see what is going on.

[Rami-Kassouf-FOO]

edited my issue to provide with hopefully a reproducable kubectl cli command

[jacobtomlinson]

edited my issue to provide with hopefully a reproducable kubectl cli command

You are importing some private code from utils.kubernetes_utils import K8, I don't know what this is doing, so I'm not able to reproduce the problem.

[RamiKassouf]

edited my issue to provide with hopefully a reproducable kubectl cli command

You are importing some private code from utils.kubernetes_utils import K8, I don't know what this is doing, so I'm not able to reproduce the problem.

I will send you the init and what the get service function does.

class K8:
    """
    A utility class for interacting with a Kubernetes cluster using the kr8s library.
    Attributes:
        api (kr8s.Api): The API client for interacting with the Kubernetes cluster.
    Methods:
        __init__():
            Initializes the K8 class and sets up the API client.
        create_namespace(namespace_name):
            Creates a namespace in the Kubernetes cluster.
        create_pod(pod_name, namespace):
            Creates a pod in the specified namespace in the Kubernetes cluster.
        get_pod(pod_name, namespace):
            Retrieves a pod by name from the specified namespace in the Kubernetes cluster.
        get_pod_by_partial_name(pod_name, namespace):
            Retrieves a pod by partial name from the specified namespace in the Kubernetes cluster.
        start_proxy():
        stop_proxy(process):
        get_secret(secret_name, namespace):
            Retrieves a secret by name from the specified namespace in the Kubernetes cluster.
        create_tls_secret(secret_name, namespace, cert_path, key_path):
            Creates a TLS secret in the specified namespace in the Kubernetes cluster.
        apply(file_path, namespace):
            Applies a Kubernetes resource file in the specified namespace in the cluster.
        get_service(service_name, namespace):
            Retrieves a service by name from the specified namespace in the Kubernetes cluster.
    """

    api: kr8s.Api

    def __init__(self):
        self.api = kr8s.api()

    def get_service(self, service_name, namespace):
        """Get a service in the Kubernetes cluster."""
        service = kr8s.objects.Service.get(service_name, namespace, api=self.api)
        logger.debug("Retrieved service %s in namespace %s", service_name, namespace)
        return service

[jacobtomlinson]

Could I ask you to just provide me with a small snippet of code that I can copy/paste in one go and reproduce the error?

[RamiKassouf]

import requests
import kr8s

payload = {"username": "admin", "password": "---"}
headers = {"Content-Type": "application/json"}

with kr8s.objects.Service.get("argocd-server", namespace="argo", api=kr8s.api()).portforward(remote_port=80, local_port="auto") as local_port:
    url = f"http://127.0.0.1:{local_port}/api/v1/session"
    response = requests.post(url, json=payload, headers=headers, verify=False)
    print(response.status_code, response.text)

That should be the equivalent i think. I m not able to test it atm but i basically replaced the K8().get_service function whith the internal implementation of it

[jacobtomlinson]

Ok thanks. I've paired things all the way down to this minimal reproducer.

# Create a test cluster and install argo
kind create -n argo-kind --image kindest/node:v1.33.0
kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

import kr8s, requests

with kr8s.objects.Service.get("argocd-server", namespace="argocd").portforward(80, "auto") as local_port:
    requests.get(f"http://127.0.0.1:{local_port}/")

ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

Full Traceback

---------------------------------------------------------------------------
RemoteDisconnected                        Traceback (most recent call last)
File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
    788 # Make the request on the HTTPConnection object
--> 789 response = self._make_request(
    790     conn,
    791     method,
    792     url,
    793     timeout=timeout_obj,
    794     body=body,
    795     headers=headers,
    796     chunked=chunked,
    797     retries=retries,
    798     response_conn=response_conn,
    799     preload_content=preload_content,
    800     decode_content=decode_content,
    801     **response_kw,
    802 )
    804 # Everything went great!

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connectionpool.py:536, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
    535 try:
--> 536     response = conn.getresponse()
    537 except (BaseSSLError, OSError) as e:

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connection.py:464, in HTTPConnection.getresponse(self)
    463 # Get the response from http.client.HTTPConnection
--> 464 httplib_response = super().getresponse()
    466 try:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:1428, in HTTPConnection.getresponse(self)
   1427 try:
-> 1428     response.begin()
   1429 except ConnectionError:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:331, in HTTPResponse.begin(self)
    330 while True:
--> 331     version, status, reason = self._read_status()
    332     if status != CONTINUE:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:300, in HTTPResponse._read_status(self)
    297 if not line:
    298     # Presumably, the server closed the connection before
    299     # sending a valid response.
--> 300     raise RemoteDisconnected("Remote end closed connection without"
    301                              " response")
    302 try:

RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

ProtocolError                             Traceback (most recent call last)
File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/adapters.py:667, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
    666 try:
--> 667     resp = conn.urlopen(
    668         method=request.method,
    669         url=url,
    670         body=request.body,
    671         headers=request.headers,
    672         redirect=False,
    673         assert_same_host=False,
    674         preload_content=False,
    675         decode_content=False,
    676         retries=self.max_retries,
    677         timeout=timeout,
    678         chunked=chunked,
    679     )
    681 except (ProtocolError, OSError) as err:

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connectionpool.py:843, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
    841     new_e = ProtocolError("Connection aborted.", new_e)
--> 843 retries = retries.increment(
    844     method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
    845 )
    846 retries.sleep()

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/util/retry.py:474, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
    473 if read is False or method is None or not self._is_method_retryable(method):
--> 474     raise reraise(type(error), error, _stacktrace)
    475 elif read is not None:

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/util/util.py:38, in reraise(tp, value, tb)
     37 if value.__traceback__ is not tb:
---> 38     raise value.with_traceback(tb)
     39 raise value

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
    788 # Make the request on the HTTPConnection object
--> 789 response = self._make_request(
    790     conn,
    791     method,
    792     url,
    793     timeout=timeout_obj,
    794     body=body,
    795     headers=headers,
    796     chunked=chunked,
    797     retries=retries,
    798     response_conn=response_conn,
    799     preload_content=preload_content,
    800     decode_content=decode_content,
    801     **response_kw,
    802 )
    804 # Everything went great!

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connectionpool.py:536, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
    535 try:
--> 536     response = conn.getresponse()
    537 except (BaseSSLError, OSError) as e:

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/urllib3/connection.py:464, in HTTPConnection.getresponse(self)
    463 # Get the response from http.client.HTTPConnection
--> 464 httplib_response = super().getresponse()
    466 try:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:1428, in HTTPConnection.getresponse(self)
   1427 try:
-> 1428     response.begin()
   1429 except ConnectionError:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:331, in HTTPResponse.begin(self)
    330 while True:
--> 331     version, status, reason = self._read_status()
    332     if status != CONTINUE:

File ~/miniconda3/envs/kr8s/lib/python3.12/http/client.py:300, in HTTPResponse._read_status(self)
    297 if not line:
    298     # Presumably, the server closed the connection before
    299     # sending a valid response.
--> 300     raise RemoteDisconnected("Remote end closed connection without"
    301                              " response")
    302 try:

ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
Cell In[19], line 4
      1 import kr8s, requests
      3 with kr8s.objects.Service.get("argocd-server", namespace="argocd").portforward(80, "auto") as local_port:
----> 4     requests.get(f"http://127.0.0.1:{local_port}/")

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/api.py:73, in get(url, params, **kwargs)
     62 def get(url, params=None, **kwargs):
     63     r"""Sends a GET request.
     64 
     65     :param url: URL for the new :class:`Request` object.
   (...)
     70     :rtype: requests.Response
     71     """
---> 73     return request("get", url, params=params, **kwargs)

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/api.py:59, in request(method, url, **kwargs)
     55 # By using the 'with' statement we are sure the session is closed, thus we
     56 # avoid leaving sockets open which can trigger a ResourceWarning in some
     57 # cases, and look like a memory leak in others.
     58 with sessions.Session() as session:
---> 59     return session.request(method=method, url=url, **kwargs)

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/sessions.py:589, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    584 send_kwargs = {
    585     "timeout": timeout,
    586     "allow_redirects": allow_redirects,
    587 }
    588 send_kwargs.update(settings)
--> 589 resp = self.send(prep, **send_kwargs)
    591 return resp

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/sessions.py:703, in Session.send(self, request, **kwargs)
    700 start = preferred_clock()
    702 # Send the request
--> 703 r = adapter.send(request, **kwargs)
    705 # Total elapsed time of the request (approximately)
    706 elapsed = preferred_clock() - start

File ~/miniconda3/envs/kr8s/lib/python3.12/site-packages/requests/adapters.py:682, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
    667     resp = conn.urlopen(
    668         method=request.method,
    669         url=url,
   (...)
    678         chunked=chunked,
    679     )
    681 except (ProtocolError, OSError) as err:
--> 682     raise ConnectionError(err, request=request)
    684 except MaxRetryError as e:
    685     if isinstance(e.reason, ConnectTimeoutError):
    686         # TODO: Remove this in 3.0.0: see #2811

ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

Notably if I use a different Pod like nginx I don't see this error. So it must be something specific to the kind of connection the Argo container is serving.

[RamiKassouf]

Yeah and the thing is it would normally work manually without kr8s.

[jacobtomlinson]

Yep I can confirm that it works with normal kubectl port-forward.