Large list rendering in JavaScript

Question

I am trying to render the list based on virtual rendering concept. I am facing some minor issues, but they are not blocking the behaviour. Here is the working fiddle http://jsfiddle.net/53N36/9/ and Here are my problems

1. Last items are not visible, I assume some where I missed indexing.(Fixed, Please see the edit)
2. How to calculate scrollPosition if I want to add custom scroll to this.
3. Is this the best method or any other?

I have tested it with 700000 items and 70 items in chrome. Below is the code

(function () {
var list = (function () {
    var temp = [];
    for (var i = 0, l = 70; i < l; i++) {
        temp.push("list-item-" + (i + 1));
    }
    return temp;
}());

function listItem(text, id) {
    var _div = document.createElement('div');
    _div.innerHTML = text;
    _div.className = "listItem";
    _div.id = id;
    return _div;
}
var listHold = document.getElementById('listHolder'),
    ht = listHold.clientHeight,
    wt = listHold.clientWidth,
    ele = listItem(list[0], 'item0'),
    frag = document.createDocumentFragment();
listHold.appendChild(ele);
var ht_ele = ele.clientHeight,
    filled = ht_ele,
    filledIn = [0];
for (var i = 1, l = list.length; i < l; i++) {
    if (filled + ht_ele < ht) {
        filled += ht_ele;
        ele = listItem(list[i], 'item' + i);
        frag.appendChild(ele);
    } else {
        filledIn.push(i);
        break;
    }
}
listHold.appendChild(frag.cloneNode(true));
var elements = document.querySelectorAll('#listHolder .listItem');

function MouseWheelHandler(e) {
    var e = window.event || e;
    var delta = Math.max(-1, Math.min(1, (e.wheelDelta || -e.detail)));
    console.log(delta);
    //if(filledIn[0] != 0 && filledIn[0] != list.length){
    if (delta == -1) {
        var start = filledIn[0] + 1,
            end = filledIn[1] + 1,
            counter = 0;
        if (list[start] && list[end]) {
            for (var i = filledIn[0]; i < filledIn[1]; i++) {
                if (list[i]) {
                    (function (a) {
                        elements[counter].innerHTML = list[a];
                    }(i));
                    counter++;
                }
            }
            filledIn[0] = start;
            filledIn[1] = end;
        }
    } else {
        var start = filledIn[0] - 1,
            end = filledIn[1] - 1,
            counter = 0;
        if (list[start] && list[end]) {
            for (var i = start; i < end; i++) {
                if (list[i]) {
                    (function (a) {
                        elements[counter].innerHTML = list[a];
                    }(i));
                    counter++;
                }
            }
            filledIn[0] = start;
            filledIn[1] = end;
        }
    }
    //}
}
if (listHold.addEventListener) {
    listHold.addEventListener("mousewheel", MouseWheelHandler, false);
    listHold.addEventListener("DOMMouseScroll", MouseWheelHandler, false);
} else listHold.attachEvent("onmousewheel", MouseWheelHandler);
}());

Please suggest me on this.

EDIT: I have tried again and I am able to fix the indexing issue. http://jsfiddle.net/53N36/26/ But how can I calculate the scroll position based on the array list currently displayed.

Gerome

Answer 1

Is this the best method or any other?
I think something that would make this much easier is not to try to handle scrolling yourself.
In this fiddle I show that you can let the browser handle scrolling for you, even though we are using virtual rendering.

Using .scrollTop I detect where the browser thinks the user is looking, and I draw in items based on that.
You'll note that if you set hidescrollbar to false and the user uses it to scroll, my method still runs fine.

Therefore, to calculate scroll position you can just use .scrollTop.
And as for custom scrolling, just make sure you influence the .scrollTop of #listHolder and recall refreshWindow()

CODE FROM FIDDLE

(function () {
    //CHANGE THESE IF YOU WANT
    var hidescrollbar = false;
    var numberofitems = 700000;
    //

    var holder = document.getElementById('listHolder');
    var view = null;

    //get the height of a single item
    var itemHeight = (function() {
        //generate a fake item
        var div = document.createElement('div');
        div.className = 'listItem';
        div.innerHTML = 'testing height';
        holder.appendChild(div);

        //get its height and remove it
        var output = div.offsetHeight;
        holder.removeChild(div);
        return output;
    })();

    //faster to instantiate empty-celled array
    var items = Array(numberofitems);
    //fill it in with data
    for (var index = 0; index < items.length; ++index)
        items[index] = 'item-' + index;

    //displays a suitable number of items
    function refreshWindow() {
        //remove old view
        if (view != null)
            holder.removeChild(view);
        //create new view
        view = holder.appendChild(document.createElement('div'));

        var firstItem = Math.floor(holder.scrollTop / itemHeight);
        var lastItem = firstItem + Math.ceil(holder.offsetHeight / itemHeight) + 1;
        if (lastItem + 1 >= items.length)
            lastItem = items.length - 1;

        //position view in users face
        view.id = 'view';
        view.style.top = (firstItem * itemHeight) + 'px';

        var div;
        //add the items
        for (var index = firstItem; index <= lastItem; ++index) {
            div = document.createElement('div');
            div.innerHTML = items[index];
            div.className = "listItem";
            view.appendChild(div);
        }
        console.log('viewing items ' + firstItem + ' to ' + lastItem);
    }

    refreshWindow();

    document.getElementById('heightForcer').style.height = (items.length * itemHeight) + 'px';
    if (hidescrollbar) {
        //work around for non-chrome browsers, hides the scrollbar
        holder.style.width = (holder.offsetWidth * 2 - view.offsetWidth) + 'px';
    }

    function delayingHandler() {
        //wait for the scroll to finish
        setTimeout(refreshWindow, 10);
    }
    if (holder.addEventListener)
        holder.addEventListener("scroll", delayingHandler, false);
    else
        holder.attachEvent("onscroll", delayingHandler);
}());

<div id="listHolder">
    <div id="heightForcer"></div>
</div>

html, body {
    width:100%;
    height:100%;
    padding:0;
    margin:0
}
body{
    overflow:hidden;
}
.listItem {
    border:1px solid gray;
    padding:0 5px;
    width: margin : 1px 0px;
}
#listHolder {
    position:relative;
    height:100%;
    width:100%;
    background-color:#CCC;
    box-sizing:border-box;
    overflow:auto;
}
/*chrome only
#listHolder::-webkit-scrollbar{
    display:none;
}*/
#view{
    position:absolute;
    width:100%;
}

Anglicism

Could you please look into this question #18286260. I am using the same code, but for horizontal list. Could you please answer – Gerome

This is very neat. My concern is that it requires consistent heights on the elements... but maybe that's just how virtual rendering has to work. – Xylina

well you could have varying heights, but depending on how you calculate the heights of the elements it could be very slow to evaluate the total height of a set of elements. Here it's a simple multiplication of a constant, but if they are variable you'd have to iteratively summate – Anglicism

whilst I appreciate the dissemination of a dedicated and supported library over my 2-hour hack, please be more courteous than "or just use" and recognise this library didn't exist at all when the question was answered. Maybe a less dismissive "There is now a cool library for this" would be more in line with SO's values. Further to that point, to alleviate any concerns regarding advertising, it would be best to disclose you are the creator clusterize @Denis – Anglicism

Wow, I didn't mean to offend you @Hashbrown. Sorry for that. That's correct, I am creator of this plugin – Yaroslavl

nah no worries, I may have misinterpreted it as dismissive because it's a comment. Seriously, you should make another answer and whack it in there, that way people are more likely to see it. I'll even chuck it an upvote because it does look neat – Anglicism

Answer 2

Thanks for the code Hashbrown, it really helped out.. Giving back to the community based on your code I tried to create a complete log viewer with rendering, highlighting, filterin, search and update capabilitiés.. It is available on this fiddle. An example of the use of the class on the fiddle is

class DisplayLogs {
    constructor(id, logs, hidescrollbar = false, emphasis = "", filter = "", render = "", openatend = false) {
        const temp = document.getElementById(id);

        this.view = null;
        this.logs = (typeof (logs) === "string" ? [logs] : logs);
        this.emphasis = emphasis;
        this.filter = filter;
        this.render = render;
        this.tab = 0;
        this.lastScrollTop = 0;
        this.hidescrollbar = hidescrollbar;
        this.highlight = { tab: -1, line: -1 };
        this.nitems = this.numberOfItems();
        this.openatend = openatend;

        temp.textContent = ""; // Clear existing content
        if (this.logs.length === 1) {
            let el = temp.appendChild(document.createElement("div"));
            el.id = "logpane";
            el.setAttribute("style", "height: 500px; position: relative; overflow-x: auto; overflow-y: auto;");
            el.appendChild(document.createElement("div")).id = "heightForcer";
            el = el.appendChild(document.createElement("div"));
            el.id = "log0";
            el.className = "container tab-pane active";
        } else {
            let el = temp.appendChild(document.createElement("ul"));
            el.className = "nav nav-tabs";
            el.id = "logs";
            el.setAttribute("role", "tablist");
            for (let i = 0; i < this.logs.length; i += 1) {
                const li = el.appendChild(document.createElement("li"));
                li.className = "nav-item";
                const link = li.appendChild(document.createElement("a"));
                link.className = "nav-link" + (i === 0 ? " active" : "");
                link.id = "navlog" + i;
                link.setAttribute("data-bs-toggle", "tab");
                link.href = "#log" + i;
                link.textContent = i;
            }
            el = temp.appendChild(document.createElement("div"));
            el.id = "logpane";
            el.setAttribute("style", "height: 500px; position: relative; overflow-x: auto; overflow-y: auto;");
            el.appendChild(document.createElement("div")).id = "heightForcer";
            for (let i = 0; i < this.logs.length; i += 1) {
                const div = el.appendChild(document.createElement("div"));
                div.id = "log" + i;
                div.className = "container tab-pane " + (i === 0 ? "active" : "fade");
            }
        }
        this.initHolder();
    }

    initHolder() {
        let i;
        this.holder = document.getElementById("logpane");
        this.height = this.itemHeight();
        if (this.holder && this.height !== 0) {
            if (this.openatend) {
                // Everything needs to be rendered with possible x-axis
                // scroll before really moving to the end
                setTimeout(this.scrollToEnd.bind(this), 75);
            }
            this.refreshWindow();
            if (this.holder.addEventListener) {
                this.holder.addEventListener("scroll", this.delayingHandler.bind(this), false);
                if (this.logs.length > 1) {
                    for (i = 0; i < this.logs.length; i += 1) { document.getElementById("navlog" + i).addEventListener("click", this.changeTab.bind(this), false); }
                }
            } else {
                this.holder.attachEvent("onscroll", this.delayingHandler.bind(this));
                if (this.logs.length > 1) {
                    for (i = 0; i < this.logs.length; i += 1) { document.getElementById("navlog" + i).attachEvent("click", this.changeTab.bind(this)); }
                }
            }
        } else { window.requestAnimationFrame(this.initHolder.bind(this)); }
    }

    scrollToEnd() {
        if (this.holder.scrollTop < this.holder.scrollHeight - this.holder.clientHeight) {
            // Don't need to explictly refresh as the event listener will deal with the scroll
            this.holder.scrollTop = this.holder.scrollHeight - this.holder.clientHeight;
        }
    }

    delayingHandler() {
        if (this.holder) {
            // Don't force refresh if scrolling in the X
            if (this.holder.scrollTop === this.lastScrollTop) { return; }
            this.lastScrollTop = this.holder.scrollTop;
        }
        setTimeout(this.refreshWindow.bind(this), 10);
    }

    changeTab(e) {
        let tab = parseInt(e.target.id.substr(6), 10);
        if (tab > this.logs.length - 1) { tab = this.logs.length - 1; }
        if (tab < 0) { tab = 0; }
        this.tab = tab;
        this.nitems = this.numberOfItems();
        this.refreshWindow();
    }

    itemHeight() {
        const pre = document.createElement("pre");
        pre.textContent = "testing height";
        this.holder.appendChild(pre);

        const output = pre.offsetHeight;
        this.holder.removeChild(pre);
        return output;
    }

    numberOfItems() {
        let output = 0;
        if (this.logs.length === 0) {
            output = 0;
        } else {
            const lines = this.logs[this.tab].split("\n");
            if (this.filter) {
                this.logs[this.tab].split("\n").forEach((line) => {
                    if (line.trim() && this.filter(line)) { output += 1; }
                });
            } else if (lines[lines.length - 1].trim()) {
                output = lines.length;
            } else {
                output = lines.length - 1;
            }
        }
        return output;
    }

    appendlog(logs, tab = 0) {
        if (logs) {
            this.logs[tab] += logs;
            if (this.tab === tab && (this.curItem
                    + Math.ceil(this.holder.offsetHeight / this.height) >= this.nitems)) {
                this.nitems = this.numberOfItems();
                this.refreshWindow();
                this.holder.scrollTop = this.holder.scrollHeight - this.holder.clientHeight;
            } else if (this.tab === tab) { this.nitems = this.numberOfItems(); }
        }
    }

    search(str = "") {
        if (str !== "") {
            const curIndex = (this.highlight.line < 0 ? this.curItem : this.highlight.line);
            const curTab = (this.highlight.tab < 0 ? this.tab : this.highlight.tab);
            const matches = [];
            let found = 0;
            for (let i = 0; i < this.logs.length; i += 1) {
                let j = 0;
                this.logs[i].split("\n").forEach((line) => {
                    if (!this.filter || this.filter(line)) {
                        if (line.includes(str)) {
                            matches.push({ tab: i, line: j });
                        }
                        j += 1;
                    }
                });
            }
            for (let j = 0; j < matches.length; j += 1) {
                if ((matches[j].tab > curTab) || ((matches[j].tab === curTab)
                    && (matches[j].line > curIndex))) {
                    found = j;
                    break;
                }
            }

            if (matches.length > 0) {
                this.tab = matches[found].tab;
                this.highlight = { tab: this.tab, line: matches[found].line };
                if (this.tab !== curTab) {
                    // If possible return focus to the element after changing tabs
                    // Allows repeated searches after changing tabs
                    if (document.activeElement) {
                        const act = document.activeElement;
                        document.querySelector("#navlog" + this.tab).click();
                        act.focus();
                    } else {
                        document.querySelector("#navlog" + this.tab).click();
                    }
                    this.nitems = this.numberOfItems();
                }
                this.refreshWindow();
                this.holder.scrollTop = Math.floor(matches[found].line * this.height);
            }
        }
    }

    updatefilter(filter = "") {
        this.filter = filter;
        this.nitems = this.numberOfItems();
        // FIXME : Rather than reset, try to keep same line
        this.highlight = { tab: this.highlight.tab, line: -1 };
        this.refreshWindow();
    }

    color(line) {
        const colors = ["text-muted", "text-dark", "text-info", "text-primary",
            "text-success", "text-warning", "text-danger"];
        let index = (this.emphasis ? this.emphasis(line) : 0);
        index = (index < 0 ? 0 : index);
        index = (index >= colors.length ? colors.length - 1 : index);
        return colors[index];
    }

    refreshWindow() {
        if (this.view != null) { this.view.remove(); }
        if (this.logs.length > 1) {
            this.view = document.getElementById("log" + this.tab).appendChild(document.createElement("div"));
        } else {
            this.view = this.holder.appendChild(document.createElement("div"));
        }

        if (this.logs.length > 0) {
            let pre;
            if (this.logs[this.tab].length > 0) {
                let lines;
                let index;
                let firstItem;
                if (this.openatend) {
                    if (this.nitems <= Math.ceil(this.holder.offsetHeight / this.height)) {
                        firstItem = 0;
                    } else {
                        firstItem = this.nitems - Math.ceil(this.holder.offsetHeight / this.height);
                    }
                } else {
                    firstItem = Math.floor(this.holder.scrollTop / this.height);
                }
                let lastItem = firstItem + Math.ceil(this.holder.offsetHeight / this.height);
                if (lastItem > this.nitems - 1) { lastItem = this.nitems - 1; }
                this.view.id = "view";
                this.view.style.top = (firstItem * this.height) + "px";
                this.view.style.position = "absolute";
                this.curItem = firstItem;

                if (this.filter) {
                    let line = 0;
                    lines = this.logs[this.tab].split("\n");
                    for (index = 0; index < lines.length; index += 1) {
                        if (this.filter(lines[index])) {
                            if (line >= firstItem) {
                                pre = document.createElement("pre");
                                if ((this.tab === this.highlight.tab)
                                        && (line === this.highlight.line)) {
                                    pre.className = "my-0 bg-info overflow-auto";
                                } else {
                                    pre.className = "my-0 " + this.color(lines[index]) + " overflow-auto";
                                }
                                pre.textContent = (this.render
                                    ? this.render(lines[index])
                                    : lines[index]);
                                this.view.appendChild(pre);
                            }
                            line += 1;
                            if (line > lastItem) { break; }
                        }
                    }
                } else {
                    lines = this.logs[this.tab].split("\n");
                    for (index = firstItem; index <= lastItem; index += 1) {
                        pre = document.createElement("pre");
                        if ((this.tab === this.highlight.tab) && (index === this.highlight.line)) {
                            pre.className = "my-0 bg-info overflow-auto";
                        } else {
                            pre.className = "my-0 " + this.color(lines[index]) + " overflow-auto";
                        }
                        pre.textContent = (this.render ? this.render(lines[index]) : lines[index]);
                        this.view.appendChild(pre);
                    }
                }
            } else {
                pre = document.createElement("pre");
                pre.className = "my-0 text-muted";
                pre.textContent = "<Empty>"; // Can't translate this easily
                this.view.appendChild(pre);
            }
        }
        // Be careful of presence or absence of x-axis scroll bar, by checking
        // against scrollHeight.
        let hf = ((this.nitems === 0 ? 1 : this.nitems) * this.height);
        if (hf < this.holder.scrollHeight) hf += this.height;
        document.getElementById("heightForcer").style.height = hf + "px";
        if (this.hidescrollbar) {
            // work around for non chrome browsers, hides the scrollbar
            this.holder.style.width = (this.holder.offsetWidth * 2 - this.view.offsetWidth) + "px";
        }
        if (this.openatend) {
            // This won't force a rerendering as the scroll
            // event listener isn't in place yet.
            this.openatend = false;
            this.holder.scrollTop = this.scrollHeight - this.clientHeight;
        }
    }
}

// The logs can be filtered to only display relevant log lines
function logFilter(line) {
    return (parseInt(line.split(' ')[0], 10) % 2 === 0);
}

// The logs can be highlighted with 7 different colours
function logHighlight(line) {
    // return 0;
    return (parseInt(line.split(' ')[0], 10) % 7);
}

// The log filter can be updated
function dsasToggleLogs() {
    const btn = document.getElementById("loghide");
    if (btn.value === "All logs") {
        btn.value = "Even logs only";
        logs.updatefilter(logFilter);
    } else {
        btn.value = "All logs";
        logs.updatefilter("");
    }
}

// Logs can be appended and the log view will stay at the
// end of the log to follow the new log entries
function appendlog(n) {
  let str = "";
  for (let j = 0; j < 10; j+=1)
    str += (n + j).toString().padEnd(10) + "   Log line\n";
  logs.appendlog(str);
  setTimeout(appendlog, 5000, n + 10);
}

const numlines = 10000;
let logfiles = [];
for  (let i = 0; i < 4; i+=1) {
  str=""
  for (let j = 0; j < numlines; j+=1)
    str += j.toString().padEnd(10) + "   Log line\n";
  logfiles.push(str);
}

document.getElementById("loghide").addEventListener("click", dsasToggleLogs);
document.getElementById("logsearch").addEventListener("keypress", (event) => {
    if (event.key === "Enter") logs.search(document.getElementById("logsearch").value);
});
logs = new DisplayLogs("logwind", logfiles, false, logHighlight, "", "", true);
setTimeout(appendlog, 5000, numlines);

<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>


<div class="container p-3 border">
  <div class="row">
    <div class="col-md-4">
      <h5>Logs :</h5>
    </div>
    <div class="col-md-8 text-end">
      <input type="button" class="btn btn-primary btn-sm" id="loghide" value="All logs">
      <input type="search" class="input-lg rounded"  id="logsearch" placeholder="Search">
    </div>
  </div>
  <span id="logwind"></span>
</div>

Stopcock