From 27f0befe3dc414b829274a33bf53cee5eac63e50 Mon Sep 17 00:00:00 2001 From: Charlotte Van Petegem Date: Thu, 8 Feb 2024 15:46:51 +0100 Subject: [PATCH] Get started on web-based assessment --- bibliography.bib | 76 ++++++++++++++++++++++++++++++++++++++++++++---- book.org | 41 ++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/bibliography.bib b/bibliography.bib index 60d26ae..015a6cf 100644 --- a/bibliography.bib +++ b/bibliography.bib @@ -376,6 +376,25 @@ file = {/home/charlotte/sync/Zotero/storage/IMAEDND9/Bennedsen and Caspersen - 2004 - Programming in context a model-first approach to .pdf} } +@article{berners-leeWorldWideWeb1992, + title = {World-{{Wide Web}}: {{The Information Universe}}}, + shorttitle = {World-{{Wide Web}}}, + author = {Berners-Lee, Tim and Cailliau, Robert and Groff, Jean-Fran{\c c}ois and Pollermann, Bernd}, + year = {1992}, + month = jan, + journal = {Internet Research}, + volume = {2}, + number = {1}, + pages = {52--58}, + publisher = {{MCB UP Ltd}}, + issn = {1066-2243}, + doi = {10.1108/eb047254}, + url = {https://doi.org/10.1108/eb047254}, + urldate = {2024-02-08}, + abstract = {The World-Wide Web (W3) initiative is a practical project designed to bring a global information universe into existence using available technology. This article describes the aims, data model, and protocols needed to implement the ``web'' and compares them with various contemporary systems.}, + file = {/home/charlotte/sync/Zotero/storage/D4DFLNVS/10.1108@eb047254.pdf.pdf;/home/charlotte/sync/Zotero/storage/6X7DIDMX/html.html} +} + @article{berniusMachineLearningBased2022, title = {Machine Learning Based Feedback on Textual Student Answers in Large Courses}, author = {Bernius, Jan Philip and Krusche, Stephan and Bruegge, Bernd}, @@ -1469,7 +1488,7 @@ googlebooks = {mOwxEAAAQBAJ}, isbn = {978-0-357-88415-7}, langid = {english}, - keywords = {Computers / Software Development \& Engineering / General} + keywords = {Computers / Software Development & Engineering / General} } @article{grgic-hlacaCaseProcessFairness2018, @@ -1775,6 +1794,18 @@ file = {/home/charlotte/sync/Zotero/storage/XVVKJK4A/Isaacson and Scott - 1989 - Automating the execution of student programs.pdf} } +@article{isong2001developing, + title = {Developing an Automated Program Checker}, + author = {Isong, Julia}, + year = {2001}, + journal = {Journal of Computing Sciences in Colleges}, + volume = {16}, + number = {3}, + pages = {218--224}, + publisher = {{Citeseer}}, + file = {/home/charlotte/sync/Zotero/storage/VWJUIVMV/Isong - 2001 - Developing an automated program checkers.pdf} +} + @inproceedings{jacksonGradingStudentPrograms1997, title = {Grading Student Programs Using {{ASSYST}}}, booktitle = {Proceedings of the Twenty-Eighth {{SIGCSE}} Technical Symposium on {{Computer}} Science Education}, @@ -1888,6 +1919,18 @@ file = {/home/charlotte/sync/Zotero/storage/5SJYAAFA/Jiang et al. - 2023 - Exploring the effects of SPOC-based blended learni.pdf} } +@article{jones2001grading, + title = {Grading Student Programs- a Software Testing Approach}, + author = {Jones, Edward L}, + year = {2001}, + journal = {Journal of Computing Sciences in Colleges}, + volume = {16}, + number = {2}, + pages = {185--192}, + publisher = {{Citeseer}}, + file = {/home/charlotte/sync/Zotero/storage/XWEESBAS/Jones - 2001 - Grading student programs- a software testing appro.pdf} +} + @article{kailaRedesigningObjectOrientedProgramming2016, title = {Redesigning an {{Object-Oriented Programming Course}}}, author = {Kaila, Erkki and Kurvinen, Einari and Lokkila, Erno and Laakso, Mikko-Jussi}, @@ -2273,8 +2316,6 @@ author = {Maertens, Rien and Van Petegem, Charlotte and Strijbol, Niko and Baeyens, Toon and Jacobs, Arne Carla and Dawyndt, Peter and Mesuere, Bart}, year = {2022}, journal = {Journal of Computer Assisted Learning}, - volume = {n/a}, - number = {n/a}, issn = {1365-2729}, doi = {10.1111/jcal.12662}, url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/jcal.12662}, @@ -2358,7 +2399,7 @@ copyright = {Open Access}, isbn = {978-989-758-021-5}, langid = {english}, - keywords = {{\`A}rees tem{\`a}tiques de la UPC::Ensenyament i aprenentatge::TIC's aplicades a l'educaci{\'o},{\`A}rees tem{\`a}tiques de la UPC::Inform{\`a}tica,Automatic assessment,Computer-assisted instruction,Data mining,Ensenyament assistit per ordinador,Mineria de dades,Online programming judges}, + keywords = {Arees tematiques de la UPC::Ensenyament i aprenentatge::TIC's aplicades a l'educacio,Arees tematiques de la UPC::Informatica,Automatic assessment,Computer-assisted instruction,Data mining,Ensenyament assistit per ordinador,Mineria de dades,Online programming judges}, annotation = {Accepted: 2015-06-04T08:47:03Z}, file = {/home/charlotte/sync/Zotero/storage/U3W7F48R/Mani et al. - 2014 - Better feedback for educational online judges.pdf} } @@ -2847,6 +2888,23 @@ url = {https://www.oecd-ilibrary.org/content/publication/589b283f-en} } +@misc{oreillyWhatWebDesign2007, + type = {{{SSRN Scholarly Paper}}}, + title = {What Is {{Web}} 2.0: {{Design Patterns}} and {{Business Models}} for the {{Next Generation}} of {{Software}}}, + shorttitle = {What Is {{Web}} 2.0}, + author = {O'Reilly, Tim}, + year = {2007}, + month = aug, + number = {1008839}, + address = {{Rochester, NY}}, + url = {https://papers.ssrn.com/abstract=1008839}, + urldate = {2024-02-08}, + abstract = {This paper was the first initiative to try to define Web 2.0 and understand its implications for the next generation of software, looking at both design patterns and business modes. Web 2.0 is the network as platform, spanning all connected devices; Web 2.0 applications are those that make the most of the intrinsic advantages of that platform: delivering software as a continually-updated service that gets better the more people use it, consuming and remixing data from multiple sources, including individual users, while providing their own data and services in a form that allows remixing by others, creating network effects through an architecture of participation, and going beyond the page metaphor of Web 1.0 to deliver rich user experiences.}, + langid = {english}, + keywords = {collective intelligence,data,long tail and beta,rich client,software as a service}, + file = {/home/charlotte/sync/Zotero/storage/KAZ6EHDA/O'Reilly - 2007 - What is Web 2.0 Design Patterns and Business Mode.pdf} +} + @article{osmanbegovicDataMiningApproach2012, title = {Data Mining Approach for Predicting Student Performance}, author = {Osmanbegovic, Edin and Suljic, Mirza}, @@ -3428,6 +3486,14 @@ file = {/home/charlotte/sync/Zotero/storage/34VD6QYR/Sels et al. - 2021 - TESTed programmeertaal-onafhankelijk testen van o.pdf} } +@phdthesis{shah2003web, + title = {Web-Cat: {{A}} Web-Based Center for Automated Testing}, + author = {Shah, Anuj Ramesh}, + year = {2003}, + school = {Virginia Tech}, + file = {/home/charlotte/sync/Zotero/storage/QM974S89/Shah - 2003 - Web-cat A web-based center for automated testing.pdf} +} + @article{shanaIdentifyingKeyPerformance2011, title = {Identifying Key Performance Indicators and Predicting the Result from Student Data}, author = {Shana, J. and Venkatachalam, T.}, @@ -4011,7 +4077,7 @@ url = {https://www.shs-conferences.org/articles/shsconf/abs/2020/05/shsconf_etltc2020_01004/shsconf_etltc2020_01004.html}, urldate = {2023-10-02}, abstract = {With the rapid development of information technology, programming has become a vital skill. An online judge system can be used as a programming education platform, where the daily activities of users and judges are used to generate useful learning objects (e.g., tasks, solution codes, evaluations). Intelligent software agents can utilize such objects to create an ecosystem. To implement such an ecosystem, a generic architecture that covers the whole lifecycle of data on the platform and the functionalities of an e-learning system should take into account the particularities of the online judge system. In this paper, an architecture that implements such an ecosystem based on an online judge system is proposed. The potential benefits and research challenges are discussed.}, - copyright = {{\textcopyright} The Authors, published by EDP Sciences, 2020}, + copyright = {{\copyright} The Authors, published by EDP Sciences, 2020}, langid = {english}, file = {/home/charlotte/sync/Zotero/storage/N4WMWTYP/Watanobe et al. - 2020 - Next-Generation Programming Learning Platform Arc.pdf} } diff --git a/book.org b/book.org index 7962ea5..53716b7 100644 --- a/book.org +++ b/book.org @@ -6,7 +6,7 @@ #+LATEX_CLASS_OPTIONS: [paper=240mm:170mm,parskip=half-,numbers=noendperiod,BCOR=10mm,DIV=10] #+LATEX_COMPILER: lualatex #+LATEX_HEADER: \usepackage[inline]{enumitem} -#+LATEX_HEADER: \usepackage{shellesc, luacode} +#+LATEX_HEADER: \usepackage{luacode} #+LATEX_HEADER: \begin{luacode*} #+LATEX_HEADER: function parseargv() #+LATEX_HEADER: local rep = {} @@ -194,11 +194,16 @@ The way people use computers has changed significantly, and the way assessment s Note that while the previous section was complete (as far as we could find), this section is decidedly not so. At this point, the explosion of automated assessment systems/automated grading systems for programming education had already set in. To describe all platforms would take a full dissertation in and of itself. -So from now on, we will pick and choose systems that brought new and interesting ideas that stood the test of time.[fn:: The ideas, not the platforms. As far as we know none of the platforms described in this section are still in use.] +So from now on, we will pick and choose systems that brought new and interesting ideas that stood the test of time.[fn:: +The ideas, not the platforms. +As far as we know none of the platforms described in this section are still in use. +] ACSES, by [cite/t:@nievergeltACSESAutomatedComputer1976], was envisioned as a full course for learning computer programming. They even designed it as a full replacement for a course: it was the first system that integrated both instructional texts and exercises. -Students following this course would not need personal instruction.[fn:: In the modern day, this would probably be considered a MOOC (except that it obviously wasn't an online course).] +Students following this course would not need personal instruction.[fn:: +In the modern day, this would probably be considered a MOOC (except that it obviously wasn't an online course). +] Another good example of this generation of grading systems is the system by [cite/t:@isaacson1989automating]. They describe the functioning of a UNIX shell script, that automatically e-mails students if their code did not compile, or if they had incorrect outputs. @@ -208,10 +213,16 @@ Like all assessment systems up to this point, they only focus on whether the out [cite/t:@reekTRYSystemHow1989] takes a different approach. He identifies several issues with gathering students' source files, and then compiling and executing them in the teacher's environment. -Students could write destructive code that destroys the teacher's files, or even write a clever program that alters their grades (and covers its tracks while doing so). +Students could write destructive code that destroys the teacher's files, or even write a clever program that alters their grades (and covers its tracks while doing so).[fn:: +Note that this issue is not new. +As we talked about before, this was already mentioned as a possibility by\nbsp{}[cite/t:@hollingsworthAutomaticGradersProgramming1960]. +This was, however, the first paper that tried to solve this problem. +] His TRY system therefore has the avoidance of teachers testing their students' programs as an explicit goal. Another goal was avoiding giving the inputs that the program was tested on to students. -These goals were mostly achieved using the UNIX =setuid= mechanism.[fn:: Note that students were thus using the same machine as the instructor, i.e., they were using a true multi-user system, as in common use at the time.] +These goals were mostly achieved using the UNIX =setuid= mechanism.[fn:: +Students were thus using the same machine as the instructor, i.e., they were using a true multi-user system, as in common use at the time. +] Every attempt was also recorded in a log file in the teacher's directory. Generality of programming language was achieved through intermediate build and test scripts that had to be provided by the teacher. @@ -228,6 +239,16 @@ ASSYST also added evaluation on other metrics, such as runtime or cyclomatic com :CREATED: [2024-02-06 Tue 17:29] :END: +After Tim Berners-Lee invented the web in 1989\nbsp{}[cite:@berners-leeWorldWideWeb1992], automated assessment systems also started moving to the web. +Especially with the rise of Web 2.0\nbsp{}[cite:@oreillyWhatWebDesign2007], allowing increased interactivity, this became more and more common. +Systems like the one by\nbsp{}[cite/t:@reekTRYSystemHow1989] also became impossible to use because of the rise of the personal computer.[fn:: +Mainly because the multi-user system was used less and less, but also because the primary way people interacted with a computer was no longer through the command line, but through graphical interfaces. +] + +Perhaps the most famous example of this is Web-CAT\nbsp{}[cite:@shah2003web]. +In addition to being one of the first web-based automated assessment platforms, it also asked the students to write their own tests. +The coverage that these tests achieved was part of the testing done by the platform. + *** Adding features :PROPERTIES: :CREATED: [2024-02-06 Tue 15:31] @@ -236,7 +257,7 @@ ASSYST also added evaluation on other metrics, such as runtime or cyclomatic com At this point in history, the idea of an automated assessment system is no longer new. But still, more and more new platforms were being written. -While almost all platforms support automated assessment of code submitted by students, contemporary platforms usually offer additional features such as gamification in the FPGE platform\nbsp{}[cite:@paivaManagingGamifiedProgramming2022], integration of full-fledged editors in iWeb-TD\nbsp{}[cite:@fonsecaWebbasedPlatformMethodology2023], exercise recommendations in PLearn\nbsp{}[cite:@vasyliukDesignImplementationUkrainianLanguage2023], automatic grading with JavAssess\nbsp{}[cite:@insaAutomaticAssessmentJava2018], assessment of test suites using test coverage measures in Web-CAT\nbsp{}[cite:@edwardsWebCATAutomaticallyGrading2008] and automatic hint generation in GradeIT\nbsp{}[cite:@pariharAutomaticGradingFeedback2017]. +While almost all platforms support automated assessment of code submitted by students, contemporary platforms usually offer additional features such as gamification in the FPGE platform\nbsp{}[cite:@paivaManagingGamifiedProgramming2022], integration of full-fledged editors in iWeb-TD\nbsp{}[cite:@fonsecaWebbasedPlatformMethodology2023], exercise recommendations in PLearn\nbsp{}[cite:@vasyliukDesignImplementationUkrainianLanguage2023], automatic grading with JavAssess\nbsp{}[cite:@insaAutomaticAssessmentJava2018], and automatic hint generation in GradeIT\nbsp{}[cite:@pariharAutomaticGradingFeedback2017]. ** Learning analytics and educational data mining @@ -2664,6 +2685,14 @@ Dodona is a pretty good piece of software. People use it, and like to use it, for some reason. We should probably try make sure that this is still the case in the future. +- Successful platform + - Lots of users + - Interesting data for scientific research +- Challenges + - Sustainability + - Generative AI + - ... + #+LATEX: \appendix * Pass/fail prediction feature types :PROPERTIES: