Develop Reference

Download file via BLOB link in Android webview

I have a website that is navigated from within a webview. One of the pages generates a ZIP file that is downloaded via a BLOB URL. I discovered that this is not supported by webview, so I have tried implementing this solution:
Download Blob file from Website inside Android WebViewClient
However, it is not working for me. Breakpoints in convertBase64StringToZipAndStoreIt are never hit.
UPDATE: I've found that I'm getting an HTTP 404. I've tried using blobUrl and blobUrl.substring(5) and the result is the same either way. The BLOBs are downloading fine in Chrome, though.
Webview setup:
private void launchWV() {
ActivityCompat.requestPermissions(this,
new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE},
1);
setContentView(R.layout.activity_self_service_launcher);
mWebView = (WebView) findViewById(R.id.activity_launcher_webview);
WebSettings webSettings = mWebView.getSettings();
webSettings.setBuiltInZoomControls(true);
webSettings.setSupportZoom(true);
webSettings.setJavaScriptEnabled(true);
webSettings.setJavaScriptCanOpenWindowsAutomatically(true);
webSettings.setSupportMultipleWindows(true);
webSettings.setAllowContentAccess(true);
webSettings.setAllowFileAccess(true);
webSettings.setAllowFileAccessFromFileURLs(true);
webSettings.setUserAgentString("Mozilla/5.0 (Android; X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.34 Safari/534.24" + getClientVersionInfo());
webSettings.setDomStorageEnabled(true);
webSettings.setLoadWithOverviewMode(true);
mWebView.setWebViewClient(new MyWebViewClient(this));
mWebView.setWebChromeClient(new MyWebChromeClient(this));
mWebView.addJavascriptInterface(new JavaScriptInterface(getApplicationContext()), "Android");
}
Function called from shouldOverrideUrlLoading() (only the else condition for BLOB URLs is of concern):
private boolean handleRequest(WebView view, String url) {
String filename;
if (!checkInternetConnection()) {
ShowNetworkUnavailableDialog(false);
return true;
}
else {
if (url.contains("view/mys") || url.contains("view/myy") || url.contains("blob") || url.contains("view/mye")) {
if (url.contains("view/mys")) {
filename = getResources().getString(R.string.mys_file_name).concat(".pdf");
} else if (url.contains("view/myy")) {
filename = getResources().getString(R.string.form_file_name).concat(".pdf");
} else if (url.contains("blob")) {
filename = getResources().getString(R.string.mys_file_name).concat(".zip");
} else {
filename = getResources().getString(R.string.mye_file_name).concat(".pdf");
}
if (!url.contains("blob")) {
String cookies = CookieManager.getInstance().getCookie(url);
DownloadManager.Request downloadRequest = new DownloadManager.Request(Uri.parse(url));
downloadRequest.addRequestHeader("cookie", cookies);
downloadRequest.allowScanningByMediaScanner();
downloadRequest.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED);
downloadRequest.setDestinationInExternalPublicDir(Environment.DIRECTORY_DOWNLOADS, filename);
DownloadManager dm = (DownloadManager) getSystemService(DOWNLOAD_SERVICE);
try {
dm.enqueue(downloadRequest);
} catch (SecurityException e) {
Toast.makeText(getApplicationContext(), getResources().getString(R.string.connection_unavailable), Toast.LENGTH_LONG).show();
return false;
}
} else {
String blobURL = JavaScriptInterface.getBase64StringFromBlobUrl(url);
mWebView.loadUrl(blobURL);
}
Toast.makeText(getApplicationContext(), getResources().getString(R.string.download_message), Toast.LENGTH_LONG).show();
return true;
} else if (!url.contains(getMetadata(getApplicationContext(), HOSTNAME))) {
//Navigate to external site outside of webview e.g. Help site
Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(url));
startActivity(intent);
return true;
} else if(url.contains(getResources().getString(R.string.about_url))) {
mWebView.loadUrl(url);
return false;
} else {
if (savedInstanceState == null) {
mWebView.loadUrl(url);
}
return false;
}
}
}
JavaScriptInterface class:
public class JavaScriptInterface {
private Context context;
private NotificationManager nm;
public JavaScriptInterface(Context context) {
this.context = context;
}
#JavascriptInterface
public void getBase64FromBlobData(String base64Data) throws IOException {
convertBase64StringToZipAndStoreIt(base64Data);
}
public static String getBase64StringFromBlobUrl(String blobUrl){
if(blobUrl.startsWith("blob")){
return "javascript: var xhr = new XMLHttpRequest();" +
"xhr.open('GET', '" + blobUrl.substring(5) + "', true);" +
"xhr.setRequestHeader('Content-type','application/zip');" +
"xhr.responseType = 'blob';" +
"xhr.onload = function(e) {" +
" if (this.status == 200) {" +
" var blobZip = this.response;" +
" var reader = new FileReader();" +
" reader.readAsDataURL(blobZip);" +
" reader.onloadend = function() {" +
" base64data = reader.result;" +
" Android.getBase64FromBlobData(base64data);" +
" }" +
" }" +
"};" +
"xhr.send();";
}
return "javascript: console.log('It is not a Blob URL');";
}
private void convertBase64StringToZipAndStoreIt(String base64Zip) throws IOException {
final int notificationId = 1;
String currentDateTime = DateFormat.getDateTimeInstance().format(new Date());
final File dwldsPath = new File(Environment.getExternalStoragePublicDirectory(
Environment.DIRECTORY_DOWNLOADS) + "/YourFileName_" + currentDateTime + "_.zip");
byte[] zipAsBytes = Base64.decode(base64Zip.replaceFirst("^data:application/zip;base64,", ""), 0);
FileOutputStream os;
os = new FileOutputStream(dwldsPath, false);
os.write(zipAsBytes);
os.flush();
if(dwldsPath.exists()) {
NotificationCompat.Builder b = new NotificationCompat.Builder(context, "MY_DL")
.setDefaults(NotificationCompat.DEFAULT_ALL)
.setWhen(System.currentTimeMillis())
.setSmallIcon(R.drawable.ic_launcher_background)
.setContentTitle("MY TITLE")
.setContentText("MY TEXT CONTENT");
nm = (NotificationManager) this.context.getSystemService(Context.NOTIFICATION_SERVICE);
if(nm != null) {
nm.notify(notificationId, b.build());
Handler h = new Handler();
long delayInMilliseconds = 5000;
h.postDelayed(new Runnable() {
public void run() {
nm.cancel(notificationId);
}
}, delayInMilliseconds);
}
}
}
}
One thing I know I am unclear on is what URL should be going into the call to xhr.open in the class.
I also tried using onDownloadStart with the same result.
Any insight is greatly appreciated!

cant decode a message from a Websocket

I`m trying to connect my HTML/JS client to my C# server as a part of a university project in order to allow the user real-time notification. (I just need the server to be able to send a specific user a message at any given time)
My server Is just a mock in order to implement it in my project.
I Successfully passed the handshake stage and I am trying to send a plain string from the server to the client. I read something about Encoding the message is a way that the client will not give the "One or more reserved bits are on: reserved1 = 0, reserved2 = 1, reserved3 = 1" error but without success.
How can I send primitive data through the Sockets and decode them on the client?
My server code:
while (true)
{
TcpListener sck = new TcpListener(IPAddress.Any, 7878);
sck.Start(1000);
TcpClient client = sck.AcceptTcpClient();
NetworkStream _stream = client.GetStream();
StreamReader clientStreamReader = new StreamReader(_stream);
StreamWriter clientStreamWriter = new StreamWriter(_stream);
while (true)
{
while (!_stream.DataAvailable) ;
Byte[] bytes = new Byte[client.Available];
_stream.Read(bytes, 0, bytes.Count());
String data = Encoding.UTF8.GetString(bytes);
if (Regex.IsMatch(data, "^GET"))
{
const string eol = "\r\n"; // HTTP/1.1 defines the sequence CR LF as the end-of-line marker
Byte[] response = Encoding.UTF8.GetBytes("HTTP/1.1 101 Switching Protocols" + eol
+ "Connection: Upgrade" + eol
+ "Upgrade: websocket" + eol
+ "Sec-WebSocket-Accept: " + Convert.ToBase64String(
System.Security.Cryptography.SHA1.Create().ComputeHash(
Encoding.UTF8.GetBytes(
new System.Text.RegularExpressions.Regex("Sec-WebSocket-Key: (.*)").Match(data).Groups[1].Value.Trim() + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
)
)
) + eol
+ eol);
_stream.Write(response, 0, response.Length);
}
else
{
}
}
}
My Client Code:
<script type="text/javascript">
function WebSocketTest() {
if ("WebSocket" in window) {
alert("WebSocket is supported by your Browser!");
// Let us open a web socket
var ws = new WebSocket("ws://localhost:7878");
ws.onopen = function () {
// Web Socket is connected, send data using send()
ws.send("Message to send");
alert("Message is sent...");
};
ws.onmessage = function (evt) {
var received_msg = evt.data;
alert("Message is received...");
};
ws.onclose = function () {
// websocket is closed.
alert("Connection is closed...");
};
} else {
// The browser doesn't support WebSocket
alert("WebSocket NOT supported by your Browser!");
}
}
</script>

I kept my server as above but added a send string function, and a decode message function:
public static string DecodeMessage(Byte[] bytes)
{
string incomingData = string.Empty;
byte secondByte = bytes[1];
int dataLength = secondByte & 127;
int indexFirstMask = 2;
if (dataLength == 126)
indexFirstMask = 4;
else if (dataLength == 127)
indexFirstMask = 10;
IEnumerable<byte> keys = bytes.Skip(indexFirstMask).Take(4);
int indexFirstDataByte = indexFirstMask + 4;
byte[] decoded = new byte[bytes.Length - indexFirstDataByte];
for (int i = indexFirstDataByte, j = 0; i < bytes.Length; i++, j++)
{
decoded[j] = (byte)(bytes[i] ^ keys.ElementAt(j % 4));
}
return Encoding.UTF8.GetString(decoded, 0, decoded.Length);
}
public static void SendString(string userName ,string str)
{
if (!userConnections.ContainsKey(userName))
return;
TcpClient client = userConnections[userName];
NetworkStream _stream = client.GetStream();
try
{
var buf = Encoding.UTF8.GetBytes(str);
int frameSize = 64;
var parts = buf.Select((b, i) => new { b, i })
.GroupBy(x => x.i / (frameSize - 1))
.Select(x => x.Select(y => y.b).ToArray())
.ToList();
for (int i = 0; i < parts.Count; i++)
{
byte cmd = 0;
if (i == 0) cmd |= 1;
if (i == parts.Count - 1) cmd |= 0x80;
_stream.WriteByte(cmd);
_stream.WriteByte((byte)parts[i].Length);
_stream.Write(parts[i], 0, parts[i].Length);
}
_stream.Flush();
}
catch (Exception ex)
{
Console.WriteLine("Error");
}
}
Where userConnections is: public static Dictionary userConnections = new Dictionary();
in order to maintain user - connection relation

You can use SuperWebSocket, this library sends the handshake automatically.
Server:
using SuperSocket.SocketBase;
using SuperWebSocket;
using System;
using System.Net;
using System.Net.Sockets;
namespace Jees.Library.WebSocket
{
public class WebSocket
{
WebSocketServer appServer;
public event EventHandler ServerStarted;
public event EventHandler ServerStopped;
public event EventHandler MessageReceived;
public string IP { get; } = string.Empty;
public int Port { get; } = 1337; //change this to the port you want to use
public WebSocket() => this.IP = GetLocalIPAddress(); //or set it manually
public void Start()
{
appServer = new WebSocketServer();
if (!appServer.Setup(this.IP, this.Port))
{
this.OnServerStarted(new WebSocketServerEventArgs(false));
return;
}
/* start listening */
appServer.NewMessageReceived += new SessionHandler<WebSocketSession, string>(AppServer_NewMessageReceived);
if (appServer.Start())
this.OnServerStarted(new WebSocketServerEventArgs(true));
else
{
this.OnServerStarted(new WebSocketServerEventArgs(false));
appServer = null;
appServer.Dispose();
}
}
public void Stop()
{
if (appServer != null)
{
appServer.Stop();
this.OnServerStopped(new EventArgs());
appServer = null;
appServer.Dispose();
}
}
private void AppServer_NewMessageReceived(WebSocketSession session, string message)
{
this.OnMessageReceived(new MessageReceivedEventArgs(message, session));
}
protected virtual void OnMessageReceived(EventArgs e) => this.MessageReceived?.Invoke(this, e);
protected virtual void OnServerStarted(EventArgs e) => this.ServerStarted?.Invoke(this, e);
protected virtual void OnServerStopped(EventArgs e) => this.ServerStopped?.Invoke(this, e);
private string GetLocalIPAddress()
{
var host = Dns.GetHostEntry(Dns.GetHostName());
foreach (var ip in host.AddressList)
if (ip.AddressFamily == AddressFamily.InterNetwork)
return ip.ToString();
throw new Exception("No network adapters with an IPv4 address in the system!");
}
}
public class WebSocketServerEventArgs : EventArgs
{
public WebSocketServerEventArgs(bool success) => this.Success = success;
public bool Success { get; }
}
public class MessageReceivedEventArgs : EventArgs
{
public MessageReceivedEventArgs(string message, WebSocketSession session)
{
this.Message = message;
this.Session = session;
}
public string Message { get; }
public WebSocketSession Session { get; }
}
}
Server Setup (I use a UserControl):
using DevExpress.XtraEditors;
using SuperWebSocket;
using System;
using System.Linq;
using System.Windows.Forms;
namespace WebSocketServer
{
public partial class Server : UserControl
{
WebSocket server;
WebSocketSession session;
public Server()
{
InitializeComponent();
server = new WebSocket();
server.ServerStarted += Server_ServerStarted;
server.ServerStopped += Server_ServerStopped;
server.MessageReceived += Server_MessageReceived;
}
private void Server_MessageReceived(object sender, EventArgs e)
{
MessageReceivedEventArgs eventArgs = (MessageReceivedEventArgs)e;
/* save session */
this.session = eventArgs.Session;
this.Log("SessionID: " + session.RemoteEndPoint.ToString() + "; Message: " + eventArgs.Message);
/* send back the message to the client */
this.session.Send(eventArgs.Message); //comment out if needed
}
private void Server_ServerStopped(object sender, EventArgs e)
{
this.Log("Server stopped!");
}
private void Server_ServerStarted(object sender, EventArgs e)
{
if ((e as WebSocketServerEventArgs).Success)
{
this.Log("Server started on ws://" + server.IP + ":" + server.Port + "/");
}
else
this.Log("Can't start the server!");
}
private void Log(string message)
{
/* here, this.log is a TextBox */
if (this.log.InvokeRequired)
this.log.Invoke((MethodInvoker)delegate
{
this.log.Text += DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss") + " > " + message + Environment.NewLine;
});
else
{
this.log.Text += DateTime.Now.ToString("yyyy/MM/dd HH:mm:ss") + " > " + message + Environment.NewLine;
}
}
/* a button to start the server */
private void BtnStart_Click(object sender, EventArgs e) => server.Start();
/* a button to stop the server */
private void BtnStop_Click(object sender, EventArgs e) => server.Stop();
/* a button to send a message from a TextBox to the client */
private void BtnSend_Click(object sender, EventArgs e)
{
if (this.txtMessage.Text != string.Empty)
this.SendMessage(this.txtMessage.Text);
}
private void SendMessage(string message)
{
try
{
/* use current session to send the message */
this.session.Send(message);
this.Log("Message: " + message + " sent to client!");
}
catch (Exception e)
{
this.Log(e.Message);
}
}
}
}
If you need more clarification, add a comment and I'll update my answer!

Opentok chat is running slowly according I execute session.disconnect() and session.connect()

I am new to the Opentok API and I am writing tests. In my tests I simulated disconnecting from the room and connecting again, but if I execute it at various times, the chat is runs slowly, until broken.
To init the chat I run $scope.initSession() and to disconnect I run $scope.disconnectFromSession().
I am using the Opentok.js version 2.13.2. See my following code:
var apiKey, sessionId, token;
apiKey = //my api key;
sessionId = //my sessionid;
token = //my token;
var session = null;
var publisher = null;
var stream = null;
var connectionCount = 0;
var publisherProperties = {frameRate: 7};
$scope.connected = false;
$scope.initSession = function() {
if (OT.checkSystemRequirements() == 1) {
// Initialize Session Object
session = OT.initSession(apiKey, sessionId);
createElement("publisher");
// initialize a publisher
publisher = OT.initPublisher('publisher', publisherProperties);
session.on({
streamCreated: function(event) {
console.log("EVENT streamCreated: " + event.stream.name + " - " + event.reason);
createElement("subscriber");
stream = event.stream;
session.subscribe(stream, 'subscriber');
},
streamDestroyed: function(event) {
event.preventDefault();
console.log("EVENT streamDestroyed: " + event.stream.name + " - " + event.reason);
console.log('Stream ${event.stream.name} ended because ${event.reason}.');
}
});
connectToSession();
} else {
console.log('Browser haven´t support to WebRTC');
}
}
function connectToSession() {
session.connect(token, function(err) {
if (err) {
if (err.name === "OT_NOT_CONNECTED") {
showMessage('Failed to connect. Please check your connection and try connecting again.');
} else {
showMessage('An unknown error occurred connecting. Please try again later.');
}
} else {
// publish to the session
session.publish(publisher);
$scope.connected = true;
}
});
}
function createElement(id) {
console.log(document.getElementById(id));
if (document.getElementById(id) === null) {
var divPublisher = document.createElement("div");
divPublisher.setAttribute("id", id);
document.getElementById("div-videos").appendChild(divPublisher);
}
}
$scope.disconnectFromSession = function() {
session.disconnect();
$scope.connected = false;
OT.off();
}
$scope.initSession();
I appreciate any help.

Java Server, html client, unable to send messages to server

i got a little problem with my project. I have a Server written in Java and some clients written in html/js. Connecting works somehow, but as soon as i want to send a message from the client to the server it returns an error: "Uncaught InvalidStateError: Failed to execute 'send' on 'WebSocket': Still in CONNECTING state"
Hopefully some of you awesome guys can look over my code and help me :)
Server Code:
Server.java
public class Server {
static ArrayList<Clients> clientsArrayList = new ArrayList<>();
private static int clientCount = 1;
private static int port;
private static ServerSocket ss;
private Socket socket;
private Clients clienthandler;
static boolean isRunning = true;
public Server(int port) throws IOException {
this.port = port;
setSs(new ServerSocket(port));
}
public void run() throws IOException {
while (isRunning) {
log("Listening on " + port + "...");
socket = getSs().accept();
log("Receiving client... " + socket);
BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()));
PrintWriter out = new PrintWriter(socket.getOutputStream());
String s;
while ((s = in.readLine()) != null) {
log(s);
if (s.isEmpty()) {
break;
}
}
log("Creating a new handler for this client...");
clienthandler = new Clients(socket, "Client " + clientCount, in, out);
Thread t = new Thread(clienthandler);
clientsArrayList.add(clienthandler);
log("Added to client list");
t.start();
clientCount++;
GUI.texttoclientlog();
}
}
public static ServerSocket getSs() {
return ss;
}
public static void setSs(ServerSocket ss) {
Server.ss = ss;
}
public void log(String logtext) {
System.out.println(logtext);
GUI.texttolog(logtext);
}
}
Clients.java
public class Clients implements Runnable {
private String name;
final BufferedReader in;
final PrintWriter out;
Socket socket;
boolean isloggedin;
public Clients(Socket socket, String name, BufferedReader in, PrintWriter out) {
this.out = out;
this.in = in;
this.name = name;
this.socket = socket;
this.isloggedin = true;
}
#Override
public void run() {
String received;
while (true) {
try {
// receive the string
received = in.readLine();
System.out.println(received);
GUI.messagehandler(this.getName() + ": " + received);
if (received.equals("logout")) {
this.isloggedin = false;
this.socket.close();
break;
}
this.in.close();
this.out.close();
this.out.flush();
this.out.flush();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public String getName() {
return name;
}
}
And the JS client code:
<script>
var connection;
connection = new WebSocket("ws://localhost:6788/");
console.log("connection established");
connection.onmessage = function (e) { console.log(e.data); };
connection.onopen = () => conn.send("Connection established");
connection.onerror = function (error) {
console.log("WebSocket Error" + error);
};
function Send() {
if (connection.readyState === 1) {
connection.send("test");
}
console.log("error sending");
}
</script>

The WebSocket session is established via a handshake.
Post the handshake is complete and the connection is upgraded, the server and client can send messages. Here is a sample WebSocket server in Java.
Update the clients run method to
public void run() {
int len = 0;
byte[] b = new byte[80];
while(true){
try {
len = in.read(b);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if(len!=-1){
byte rLength = 0;
int rMaskIndex = 2;
int rDataStart = 0;
//b[0] is always text in my case so no need to check;
byte data = b[1];
byte op = (byte) 127;
rLength = (byte) (data & op);
if(rLength==(byte)126) rMaskIndex=4;
if(rLength==(byte)127) rMaskIndex=10;
byte[] masks = new byte[4];
int j=0;
int i=0;
for(i=rMaskIndex;i<(rMaskIndex+4);i++){
masks[j] = b[i];
j++;
}
rDataStart = rMaskIndex + 4;
int messLen = len - rDataStart;
byte[] message = new byte[messLen];
for(i=rDataStart, j=0; i<len; i++, j++){
message[j] = (byte) (b[i] ^ masks[j % 4]);
}
System.out.println(new String(message));
b = new byte[80];
}
}
}
based on this SO answer
In my opinion you can leverage the Spring WebSocket support, rather than writing your own. I had created one which also uses ProtoBuf or you can look at the Spring WebSocket getting started guide here

Thats the updated code. it receives the bytes from the html clients. but i have no clue how to decode them :)
Server.java
public class Server {
static ArrayList<Clients> clientsArrayList = new ArrayList<>();
private static int clientCount = 1;
private static int port;
private static ServerSocket ss;
private Socket socket;
private Clients clienthandler;
static boolean isRunning = true;
private InputStream in;
private OutputStream out;
public Server(int port) throws IOException {
Server.port = port;
setSs(new ServerSocket(port));
}
public void run() throws IOException, NoSuchAlgorithmException {
while (isRunning) {
log("Listening on " + port + "...");
socket = getSs().accept();
log("Receiving client... " + socket);
in = socket.getInputStream();
out = socket.getOutputStream();
#SuppressWarnings("resource")
String data = new Scanner(in, "UTF-8").useDelimiter("\\r\\n\\r\\n").next();
Matcher get = Pattern.compile("^GET").matcher(data);
if (get.find()) {
Matcher match = Pattern.compile("Sec-WebSocket-Key: (.*)").matcher(data);
match.find();
byte[] response = ("HTTP/1.1 101 Switching Protocols\r\n" + "Connection: Upgrade\r\n"
+ "Upgrade: websocket\r\n" + "Sec-WebSocket-Accept: "
+ DatatypeConverter.printBase64Binary(MessageDigest.getInstance("SHA-1")
.digest((match.group(1) + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").getBytes("UTF-8")))
+ "\r\n\r\n").getBytes("UTF-8");
out.write(response, 0, response.length);
log("Creating a new handler for this client...");
clienthandler = new Clients(socket, "Client " + clientCount, in, out);
Thread t = new Thread(clienthandler);
clientsArrayList.add(clienthandler);
log("Added to client list");
t.start();
clientCount++;
GUI.texttoclientlog();
} else {
log("Handshake failed");
}
}
}
public static ServerSocket getSs() {
return ss;
}
public static void setSs(ServerSocket ss) {
Server.ss = ss;
}
public void log(String logtext) {
System.out.println(logtext);
GUI.texttolog(logtext);
}
}
Clients.java
public class Clients implements Runnable {
private String name;
final InputStream in;
final OutputStream out;
Socket socket;
boolean isloggedin;
public Clients(Socket socket, String name, InputStream in, OutputStream out) {
this.out = out;
this.in = in;
this.name = name;
this.socket = socket;
this.isloggedin = true;
}
#Override
public void run() {
int received;
while (true) {
try {
received = in.read();
//how to decode??
System.out.println(received);
GUI.messagehandler(this.getName() + ": " + received);
} catch (IOException e) {
e.printStackTrace();
}
}
}
public String getName() {
return name;
}
}

A console application to get a web page resource, using c# (javascript may cause this)

Aim: To download a website source with using a console application. You can find the used class in the program below.
Question: I use the code below to download a data (source) of a web page. Imagine you use chrome; If you enter first this query string, the web page itself redirects you a view HTML page and you see the data.
Entering this URL, to show the results it redirects itself to second page below. I make it by using javascript.
www.xyz.com/aaa.html?search=aaa&id=1
it redirects here: www.xyz.com/ViewResult.html
In an explorer, It works fine . I see 4 HTML tables inside the page when I use google chrome view source option. Bu in my application I see only two tables of the 4 . The two tables inside the web page is missing.(the missing two tables are the second and third.)
How can I overcome to this problem? I want to get the source of the page as I see in chrome.
Bonus informations: There is no iframe.
The particular Code :
string url = "www.xyz.com/aaa.html?search=aaa&id=1";
WebPage pG = ss.RequestPage(url, "", "GET");
pG = ss.RequestPage("www.xyz.com/ViewResult.html");
string source= pG.Html;
public WebPage RequestPage(Uri url, string content, string method, string contentType)
{
string htmlResult;
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
HttpWebResponse response = null;
ASCIIEncoding encoding = new ASCIIEncoding();
byte[] contentData = encoding.GetBytes(content);
request.Proxy = Proxy;
request.Timeout = 60000;
request.Method = method;
request.AllowAutoRedirect = false; // false
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.Referer = LastUrl;
request.KeepAlive = true; //false,
request.UserAgent = UserAgent;
request.Headers.Add("Accept-Language", "en-us,en;q=0.5");
//request.Headers.Add("UA-CPU", "x86");
request.Headers.Add("Cache-Control", "no-cache");
request.Headers.Add("Accept-Encoding", "gzip,deflate");
String cookieString = "";
foreach (KeyValuePair<String, String> cookiePair in Cookies)
cookieString += cookiePair.Key + "=" + cookiePair.Value + ";";
if (cookieString.Length > 2)
{
String cookie = cookieString.Substring(0, cookieString.Length - 1);
request.Headers.Add("Cookie", cookie);
}
if (method == "POST")
{
request.ContentLength = contentData.Length;
request.ContentType = contentType;
Stream contentWriter = request.GetRequestStream();
contentWriter.Write(contentData, 0, contentData.Length);
contentWriter.Close();
}
int attempts = 0;
while (true)
{
try
{
response = (HttpWebResponse)request.GetResponse();
if (response == null)
throw new WebException();
break;
}
catch (WebException)
{
if (response != null)
response.Close();
if (attempts == PageReattempts)
{
throw;
}
else { }
// Wait three seconds before trying again
Thread.Sleep(3000);
}
attempts += 1;
}
// Tokenize cookies
if (response.Headers["Set-Cookie"] != null)
{
String headers = response.Headers["Set-Cookie"].Replace("path=/,", ";").Replace("HttpOnly,", "");
foreach (String cookie in headers.Split(';'))
{
if (cookie.Contains("="))
{
String[] splitCookie = cookie.Split('=');
String cookieKey = splitCookie[0].Trim();
String cookieValue = splitCookie[1].Trim();
if (Cookies.ContainsKey(cookieKey))
Cookies[cookieKey] = cookieValue;
else
Cookies.Add(cookieKey, cookieValue);
}
else
{
if (Cookies.ContainsKey(cookie))
Cookies[cookie] = "";
else
Cookies.Add(cookie, "");
}
}
}
htmlResult = ReadResponseStream(response);
response.Close();
if (response.Headers["Location"] != null)
{
response.Close();
Thread.Sleep(1500);
String newLocation = response.Headers["Location"];
WebPage result = RequestPage(newLocation);
return new WebPage(result.Html, new WebPage(htmlResult));
}
LastUrl = url.ToString();
return new WebPage(htmlResult);
}

1-WebBrowser :
public class ExtendedWebBrowser : System.Windows.Forms.WebBrowser
{
public ExtendedWebBrowser()
{
// Ensure that ScriptErrorsSuppressed is set to false.
this.ScriptErrorsSuppressed = true;
this.ProgressChanged += ExtendedWebBrowser_ProgressChanged;
}
private void ExtendedWebBrowser_ProgressChanged(object sender, WebBrowserProgressChangedEventArgs e)
{
// InjectAlertBlocker();
string alertBlocker = #"window.alert = function () { };
window.print = function () { };
window.open = function () { };
window.onunload = function () { };
window.onbeforeunload = function () { };";
var webBrowser = sender as WebBrowser;
webBrowser?.Document?.InvokeScript("execScript", new Object[] { alertBlocker, "JavaScript" });
this.Document?.InvokeScript("execScript", new Object[] { alertBlocker, "JavaScript" });
}
public void NavigationWaitToComplete(string url)
{
bool complete = false;
NavigationAsync(url).ContinueWith((t) => complete = true);
while (!complete)
{
System.Windows.Forms.Application.DoEvents();
}
}
public void NavigationWaitToComplete(string url, string targetFrameName, byte[] postData, string additionalHeaders)
{
bool complete = false;
NavigationAsync(url, targetFrameName, postData, additionalHeaders).ContinueWith((t) => complete = true);
while (!complete)
{
System.Windows.Forms.Application.DoEvents();
}
}
public async Task NavigationAsync(string url, string targetFrameName, byte[] postData, string additionalHeaders)
{
TaskCompletionSource<bool> tcsNavigation = new TaskCompletionSource<bool>(); ;
TaskCompletionSource<bool> tcsDocument = new TaskCompletionSource<bool>(); ;
Navigated += (s, e) =>
{
if (tcsNavigation.Task.IsCompleted)
return;
tcsNavigation.SetResult(true);
};
DocumentCompleted += (s, e) =>
{
if (ReadyState != WebBrowserReadyState.Complete)
return;
if (tcsDocument.Task.IsCompleted)
return;
tcsDocument.SetResult(true);
};
Navigate(url, targetFrameName, postData, additionalHeaders);
await tcsNavigation.Task;
// navigation completed, but the document may still be loading
await tcsDocument.Task;
// the document has been fully loaded, you can access DOM here
}
public async Task NavigationAsync(string url)
{
TaskCompletionSource<bool> tcsNavigation = new TaskCompletionSource<bool>(); ;
TaskCompletionSource<bool> tcsDocument = new TaskCompletionSource<bool>(); ;
Navigated += (s, e) =>
{
if (tcsNavigation.Task.IsCompleted)
return;
tcsNavigation.SetResult(true);
};
DocumentCompleted += (s, e) =>
{
if (ReadyState != WebBrowserReadyState.Complete)
return;
if (tcsDocument.Task.IsCompleted)
return;
tcsDocument.SetResult(true);
};
Navigate(url);
await tcsNavigation.Task;
// navigation completed, but the document may still be loading
await tcsDocument.Task;
// the document has been fully loaded, you can access DOM here
}
}
Calling:
var browser = new ExtendedWebBrowser();
browser.NavigationWaitToComplete("www.xyz.com/aaa.html?search=aaa&id=1");
var html = browser.Document.Body.OuterHtml();
2-CefSharp.OffScreen
private async Task<string> RequestPageAsync(string url, string cachePath, double zoomLevel)
{
var tcs = new TaskCompletionSource<string>();
var browserSettings = new BrowserSettings();
//Reduce rendering speed to one frame per second so it's easier to take screen shots
browserSettings.WindowlessFrameRate = 1;
var requestContextSettings = new RequestContextSettings { CachePath = cachePath };
// RequestContext can be shared between browser instances and allows for custom settings
// e.g. CachePath
using (var requestContext = new RequestContext(requestContextSettings))
using (var browser = new ChromiumWebBrowser(url, browserSettings, requestContext))
{
if (zoomLevel > 1)
{
browser.FrameLoadStart += (s, argsi) =>
{
var b = (ChromiumWebBrowser)s;
if (argsi.Frame.IsMain)
{
b.SetZoomLevel(zoomLevel);
}
};
}
browser.FrameLoadEnd += (s, argsi) =>
{
var b = (ChromiumWebBrowser)s;
if (argsi.Frame.IsMain)
{
b.GetSourceAsync().ContinueWith(taskHtml =>
{
tcs.TrySetResult(taskHtml.Result);
});
}
};
}
return tcs.Task.Result;
}
Calling :
RequestPageAsync("www.xyz.com/aaa.html?search=aaa&id=1", "cachePath1", 1.0);