Difference between revisions of "Curl"

From UVOO Tech Wiki
Jump to navigation Jump to search
 
(26 intermediate revisions by the same user not shown)
Line 1: Line 1:
 +
```
 +
curl 'https://10.x.x.x/foo' -H 'Host: foo.example.com' -k -I -s -S
 +
```
 +
 +
```
 +
curl -vkX GET https://10.x..x.x:443/health 2>&1 | grep 200
 +
```
 +
 +
```
 +
GET /index.html HTTP/1.1\nHost: www.yourhost.com \nConnection: close\n\n
 +
```
 +
 
SNI header
 
SNI header
 
```
 
```
 +
env=qa; ip=$(dig +short lb.${env}.example.com.); host=foo.${env}.example.com; curl -sv --resolve $host:443:$ip https://$host/some/path
 +
curl -k -I
 
curl --connect-to www.example.com:443:example.a:443 https://www.example.com
 
curl --connect-to www.example.com:443:example.a:443 https://www.example.com
−
curl -sv   --resolve $host:443:$ip https://$host
+
host=api.example.com; ip=10.x.x.x; curl -sv --resolve $host:443:$ip https://$host/mypath/foo
 
curl --resolve example.com:443:127.0.0.1 https://example.com/
 
curl --resolve example.com:443:127.0.0.1 https://example.com/
 
curl -vik --resolve example.com:443:198.18.110.10 https://example.com/
 
curl -vik --resolve example.com:443:198.18.110.10 https://example.com/
 +
 +
curl.exe --connect-to api.example.com:443:myapp.azurewebsites.net https://api.example.com/mypath/health
 +
 +
curl.exe -w "%{time_total}" --connect-to api.example.com:443:myapp.azurewebsites.net https://api.example.com/mypath/health
 
```
 
```
  
Line 14: Line 32:
 
```
 
```
 
curl --header "Host: example.com" http://127.0.0.1/
 
curl --header "Host: example.com" http://127.0.0.1/
 +
```
 +
 +
```
 +
curl.exe -H "Origin: https://host.example.com" -vvv -X POST https://host.example.com/app/api/users/authorize -H 'Content-Type: application/json' -d '{"username": "foo", "password": "bar"}'
 
```
 
```
  
Line 31: Line 53:
 
for host in 10.1.1.10 10.1.1.52 10.1.1.16 10.1.1.16; do echo $host; curl --header "Host: www.example.org" http://$host/downloads/myfile.png --output /dev/null; done
 
for host in 10.1.1.10 10.1.1.52 10.1.1.16 10.1.1.16; do echo $host; curl --header "Host: www.example.org" http://$host/downloads/myfile.png --output /dev/null; done
 
```
 
```
 +
 +
# SNI Spoofing for HTTPS
 +
https://everything.curl.dev/usingcurl/connections/name
 +
 +
https://daniel.haxx.se/blog/2018/04/05/curl-another-host/
  
 
BASH Wrapper
 
BASH Wrapper
Line 59: Line 86:
 
   cmd="/usr/bin/curl -w \" %{$metric}\" -o /dev/null -s \"$url\""
 
   cmd="/usr/bin/curl -w \" %{$metric}\" -o /dev/null -s \"$url\""
 
   metric_value=$(eval "$cmd")
 
   metric_value=$(eval "$cmd")
−
elif [[ "$curltype" == "spoofresolve" ]]; then
+
elif [[ "$curltype" == "resolve" ]]; then
 
   if [[ "$#" -lt 4 ]]; then
 
   if [[ "$#" -lt 4 ]]; then
 
     echo "Usage: $0 <type> <curl metric> <url> <spoof ipaddr>"
 
     echo "Usage: $0 <type> <curl metric> <url> <spoof ipaddr>"
−
     echo "Example: $0 spoofresolve time_total https://example.org 127.0.0.1"
+
     echo "Example: $0 resolve time_total https://example.org 127.0.0.1"
 
     exit
 
     exit
 
   fi
 
   fi
Line 68: Line 95:
 
   host=$(echo "$url" | awk -F/ '{print $3}')
 
   host=$(echo "$url" | awk -F/ '{print $3}')
 
   cmd="/usr/bin/curl -w \"%{time_total}\" -o /dev/null -s --resolve $host:443:$ipaddr \"$url\""
 
   cmd="/usr/bin/curl -w \"%{time_total}\" -o /dev/null -s --resolve $host:443:$ipaddr \"$url\""
 +
  metric_value=$(eval "$cmd")
 +
elif [[ "$curltype" == "connect-to" ]]; then
 +
  if [[ "$#" -lt 4 ]]; then
 +
    echo "Usage: $0 <type> <curl metric> <url> <fqdn>"
 +
    echo "Example: $0 connect-to time_total https://example.org myapp-svc.azurewebsites.net"
 +
    exit
 +
  fi
 +
  dstadc_fqdn=$4
 +
  host=$(echo "$url" | awk -F/ '{print $3}')
 +
  cmd="/usr/bin/curl -w \"%{time_total}\" -o /dev/null -s --resolve $host:443:$dstadc_fqdn \"$url\""
 
   metric_value=$(eval "$cmd")
 
   metric_value=$(eval "$cmd")
 
else
 
else
Line 80: Line 117:
 
     echo "$metric_value"
 
     echo "$metric_value"
 
fi
 
fi
 +
```
 +
 +
 +
 +
load generator using curl
 +
```
 +
#!/usr/bin/env bash
 +
set -eu
 +
# max_time=.1
 +
max_time=2
 +
# --connect-timeout
 +
# connection_timeout=1
 +
load_url(){
 +
  url=$1
 +
  adc=$2
 +
  text=$3
 +
  count=$4
 +
  host=$(echo "$url" | awk -F/ '{print $3}')
 +
 +
  a=()
 +
  for i in $(seq $count); do
 +
    curl -m $max_time -s --resolve $host:443:$adc $url | grep -i $text > /dev/null
 +
    if [[ $? == 0 ]]; then
 +
      a+=(0)
 +
    else
 +
      a+=(1)
 +
    fi
 +
    echo $a
 +
  done
 +
}
 +
 +
count=5
 +
url="https://example.com/path/health"
 +
adc="myapp.azurewebsites.net"
 +
text="Healthy"
 +
r=$(load_url $url $adc $text $count)
 +
echo $r
 +
 +
adc="lb.example.com"
 +
r=$(load_url $url $adc $text $count)
 +
echo $r
 +
 +
 +
# for value in "${a[@]}"; do
 +
#  echo $value
 +
# done
 +
```
 +
 +
# Parallel
 +
Parallel https://unix.stackexchange.com/questions/103920/parallelize-a-bash-for-loop
 +
```
 +
xargs -I % -P 5 curl -I "https://linuxways.net" < <(printf '%s\n' {1..10})
 +
seq 1 10 | xargs -I % -P 5 curl -I "https://linuxways.net"
 +
xargs -P 5 -n 1 curl -O < download.txt
 +
 +
 +
 +
xargs -I % -P 8 curl -X POST --header "http://localhost:5000/example" \
 +
< <(printf '%s\n' {1..400})
 +
This will run give curl command 400 times with max 8 jobs in parallel.
 +
```
 +
https://linuxhint.com/execute-multiple-curl-requests-parallel/
 +
https://stackoverflow.com/questions/46362284/run-multiple-curl-commands-in-parallel
 +
 +
```
 +
parallel_curl(){
 +
  local fqdn=$1
 +
  local parallel_count=$2
 +
  local total_count=$3
 +
  local ipaddr=$4
 +
  xargs -I % -P $parallel_count curl -kIL --resolve $fqdn:443:$ipaddr https://$fqdn/ \
 +
    < <(printf '%s\n' $(seq ${total_count}))
 +
}
 +
 +
parallel_curl www.uvoo.io 20 40 192.168.1.100
 +
```
 +
 +
 +
urlloader.sh
 +
```
 +
#!/bin/bash
 +
set -eu
 +
if [[ "$#" -lt 7 ]]; then
 +
  echo "Usage: $0 <action> <entry-host> <url> <count> <tcount> <env> <text>"
 +
  echo "Example: $0 load1 appservice.azurewebsites.net  https://www.example.com/path/health 1 5 dev Healthy"
 +
  exit
 +
fi
 +
action=$1
 +
entry_host=$2
 +
url=$3
 +
count=$4
 +
tcount=$5
 +
env=$6
 +
text=$7
 +
host=$(echo "$url" | awk -F/ '{print $3}')
 +
 +
 +
get_url(){
 +
  action=$1
 +
  entry_host=$2
 +
  url=$3
 +
  count=$4
 +
  tcount=$5
 +
  env=$6
 +
  text=$7
 +
  host=$8
 +
  r=$(curl -s --resolve $host:443:$entry_host $url | grep $text > /dev/null || echo 1)
 +
  echo -n "$r "
 +
}
 +
 +
 +
load1(){
 +
  export -f get_url
 +
  start_time=$(date +%s%N)
 +
  seq $tcount  | xargs -n 1 -P $count -I {} bash -c "get_url $action $entry_host $url $count $tcount $env $text $host"
 +
  end_time=$(date +%s%N)
 +
  diff_nanoseconds=$((end_time-start_time))
 +
  diff_seconds=$(bc <<< "scale=3; $diff_nanoseconds / 1000000000")
 +
  echo ""
 +
  echo seconds:$diff_seconds
 +
}
 +
 +
if [[ $action == "load1" ]]; then
 +
  load1
 +
else
 +
  echo "E: Unsupported action!"
 +
fi
 +
 +
```
 +
 +
# Cipher Force
 +
```
 +
--ciphers ECDHE-RSA-AES256-GCM-SHA256
 +
--ciphers ECDHE-RSA-AES128-GCM-SHA256
 +
seq 30 | xargs -n 1 -P 30 bash -c "curl https://example.com/path/health --ciphers ECDHE-RSA-AES128-GCM-SHA256 --resolve example.com:443:10.x.x.x -s | grep -i he
 +
alth > /dev/null || echo 1"
 +
```
 +
 +
 +
```
 +
Create a new file, curl-format.txt, and paste in:
 +
 +
    time_namelookup:  %{time_namelookup}s\n
 +
        time_connect:  %{time_connect}s\n
 +
    time_appconnect:  %{time_appconnect}s\n
 +
    time_pretransfer:  %{time_pretransfer}s\n
 +
      time_redirect:  %{time_redirect}s\n
 +
  time_starttransfer:  %{time_starttransfer}s\n
 +
                    ----------\n
 +
          time_total:  %{time_total}s\n
 +
Make a request:
 +
 +
curl -w "@curl-format.txt" -o /dev/null -s "http://wordpress.com/"
 +
Or on Windows, it's...
 +
 +
curl -w "@curl-format.txt" -o NUL -s "http://wordpress.com/"
 
```
 
```

Latest revision as of 15:32, 11 April 2024

curl 'https://10.x.x.x/foo' -H 'Host: foo.example.com' -k -I -s -S

curl -vkX GET https://10.x..x.x:443/health 2>&1 | grep 200

GET /index.html HTTP/1.1\nHost: www.yourhost.com \nConnection: close\n\n

SNI header 

env=qa; ip=$(dig +short lb.${env}.example.com.); host=foo.${env}.example.com; curl -sv --resolve $host:443:$ip https://$host/some/path
curl -k -I 
curl --connect-to www.example.com:443:example.a:443 https://www.example.com
host=api.example.com; ip=10.x.x.x; curl -sv --resolve $host:443:$ip https://$host/mypath/foo
curl --resolve example.com:443:127.0.0.1 https://example.com/
curl -vik --resolve example.com:443:198.18.110.10 https://example.com/

curl.exe --connect-to api.example.com:443:myapp.azurewebsites.net https://api.example.com/mypath/health

curl.exe -w "%{time_total}" --connect-to api.example.com:443:myapp.azurewebsites.net https://api.example.com/mypath/health

 tshark -l -i F5_Internal -f 'dst port ( 443 )' -Y 'ssl.handshake.extension.type == "server_name" || http.host' -T fields -e ip.src -e ip.dst -e tcp.dstport -e ssl.handshake.extensions_server_name -e http.host | grep example

Non sni header (port 80 unencrypted header) 

curl --header "Host: example.com" http://127.0.0.1/

curl.exe -H "Origin: https://host.example.com" -vvv -X POST https://host.example.com/app/api/users/authorize -H 'Content-Type: application/json' -d '{"username": "foo", "password": "bar"}'

https://daniel.haxx.se/blog/2018/04/05/curl-another-host/

Testing ciphers 

# curl https://www.example.com -k -v --location-trusted --sslv2
curl https://www.example.com -k -v --location-trusted --tlsv1.1
curl https://www.example.com -k -v --location-trusted --tlsv1.2
curl https://www.example.com -k -v --location-trusted --tlsv1.3

for host in 10.1.1.10 10.1.1.52 10.1.1.16 10.1.1.16; do echo $host; curl --header "Host: www.example.org" http://$host/downloads/myfile.png --output /dev/null; done

SNI Spoofing for HTTPS

https://everything.curl.dev/usingcurl/connections/name

https://daniel.haxx.se/blog/2018/04/05/curl-another-host/

BASH Wrapper 

#!/usr/bin/env bash

if [[ "$#" -lt 3 ]]; then
  echo "Usage: $0 <type> <curl metric> <url>"
  echo "Example: $0 url time_connect https://google.com"
echo "
====URL Type Metrics====
time_namelookup
time_connect
time_appconnec
time_pretransfer
time_redirect
time_starttransfer
time_total
"
  exit
fi
curltype=$1
metric=$2
url=$3

if [[ "$curltype" == "url" ]]; then
  cmd="/usr/bin/curl -w \" %{$metric}\" -o /dev/null -s \"$url\""
  metric_value=$(eval "$cmd")
elif [[ "$curltype" == "resolve" ]]; then
  if [[ "$#" -lt 4 ]]; then
    echo "Usage: $0 <type> <curl metric> <url> <spoof ipaddr>"
    echo "Example: $0 resolve time_total https://example.org 127.0.0.1"
    exit
  fi
   ipaddr=$4
   host=$(echo "$url" | awk -F/ '{print $3}')
   cmd="/usr/bin/curl -w \"%{time_total}\" -o /dev/null -s --resolve $host:443:$ipaddr \"$url\""
   metric_value=$(eval "$cmd")
elif [[ "$curltype" == "connect-to" ]]; then
  if [[ "$#" -lt 4 ]]; then
    echo "Usage: $0 <type> <curl metric> <url> <fqdn>"
    echo "Example: $0 connect-to time_total https://example.org myapp-svc.azurewebsites.net"
    exit
  fi
   dstadc_fqdn=$4
   host=$(echo "$url" | awk -F/ '{print $3}')
   cmd="/usr/bin/curl -w \"%{time_total}\" -o /dev/null -s --resolve $host:443:$dstadc_fqdn \"$url\""
   metric_value=$(eval "$cmd")
else
  echo "E: Unsupported type of check."
fi

code=$?
if [[ "$code" != 0 ]]; then
   # echo error
    echo "$metric_value"
else
    echo "$metric_value"
fi

load generator using curl 

#!/usr/bin/env bash
set -eu
# max_time=.1
max_time=2
# --connect-timeout
# connection_timeout=1
load_url(){
  url=$1
  adc=$2
  text=$3
  count=$4
  host=$(echo "$url" | awk -F/ '{print $3}')

  a=()
  for i in $(seq $count); do
    curl -m $max_time -s --resolve $host:443:$adc $url | grep -i $text > /dev/null
    if [[ $? == 0 ]]; then
       a+=(0)
    else
       a+=(1)
    fi
    echo $a
  done
}

count=5
url="https://example.com/path/health"
adc="myapp.azurewebsites.net"
text="Healthy"
r=$(load_url $url $adc $text $count)
echo $r

adc="lb.example.com"
r=$(load_url $url $adc $text $count)
echo $r


# for value in "${a[@]}"; do
#   echo $value
# done

Parallel

Parallel https://unix.stackexchange.com/questions/103920/parallelize-a-bash-for-loop 

xargs -I % -P 5 curl -I "https://linuxways.net" < <(printf '%s\n' {1..10})
seq 1 10 | xargs -I % -P 5 curl -I "https://linuxways.net"
xargs -P 5 -n 1 curl -O < download.txt



xargs -I % -P 8 curl -X POST --header "http://localhost:5000/example" \
< <(printf '%s\n' {1..400})
This will run give curl command 400 times with max 8 jobs in parallel.

https://linuxhint.com/execute-multiple-curl-requests-parallel/ https://stackoverflow.com/questions/46362284/run-multiple-curl-commands-in-parallel 

parallel_curl(){
  local fqdn=$1
  local parallel_count=$2
  local total_count=$3
  local ipaddr=$4
  xargs -I % -P $parallel_count curl -kIL --resolve $fqdn:443:$ipaddr https://$fqdn/ \
    < <(printf '%s\n' $(seq ${total_count}))
}

parallel_curl www.uvoo.io 20 40 192.168.1.100

urlloader.sh 

#!/bin/bash
set -eu
if [[ "$#" -lt 7 ]]; then
  echo "Usage: $0 <action> <entry-host> <url> <count> <tcount> <env> <text>"
  echo "Example: $0 load1 appservice.azurewebsites.net  https://www.example.com/path/health 1 5 dev Healthy"
  exit
fi
action=$1
entry_host=$2
url=$3
count=$4
tcount=$5
env=$6
text=$7
host=$(echo "$url" | awk -F/ '{print $3}')


get_url(){
  action=$1
  entry_host=$2
  url=$3
  count=$4
  tcount=$5
  env=$6
  text=$7
  host=$8
  r=$(curl -s --resolve $host:443:$entry_host $url | grep $text > /dev/null || echo 1)
  echo -n "$r "
}


load1(){
  export -f get_url
  start_time=$(date +%s%N)
  seq $tcount  | xargs -n 1 -P $count -I {} bash -c "get_url $action $entry_host $url $count $tcount $env $text $host"
  end_time=$(date +%s%N)
  diff_nanoseconds=$((end_time-start_time))
  diff_seconds=$(bc <<< "scale=3; $diff_nanoseconds / 1000000000")
  echo ""
  echo seconds:$diff_seconds
}

if [[ $action == "load1" ]]; then
  load1
else
  echo "E: Unsupported action!"
fi

Cipher Force

--ciphers ECDHE-RSA-AES256-GCM-SHA256
--ciphers ECDHE-RSA-AES128-GCM-SHA256
seq 30 | xargs -n 1 -P 30 bash -c "curl https://example.com/path/health --ciphers ECDHE-RSA-AES128-GCM-SHA256 --resolve example.com:443:10.x.x.x -s | grep -i he
alth > /dev/null || echo 1"

Create a new file, curl-format.txt, and paste in:

     time_namelookup:  %{time_namelookup}s\n
        time_connect:  %{time_connect}s\n
     time_appconnect:  %{time_appconnect}s\n
    time_pretransfer:  %{time_pretransfer}s\n
       time_redirect:  %{time_redirect}s\n
  time_starttransfer:  %{time_starttransfer}s\n
                     ----------\n
          time_total:  %{time_total}s\n
Make a request:

 curl -w "@curl-format.txt" -o /dev/null -s "http://wordpress.com/"
Or on Windows, it's...

 curl -w "@curl-format.txt" -o NUL -s "http://wordpress.com/"
Retrieved from "https://tech.uvoo.io/index.php?title=Curl&oldid=5194"